Skip to content

Instantly share code, notes, and snippets.

@bukowa
Last active April 14, 2023 23:18
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save bukowa/4a2d5aceff4c3e146331ea03f29af785 to your computer and use it in GitHub Desktop.
Save bukowa/4a2d5aceff4c3e146331ea03f29af785 to your computer and use it in GitHub Desktop.
grafana freqtrade dashboard
its WIP but works, just
1. run grafana if u dont have it docker run --rm -it -e GF_SECURITY_ADMIN_PASSWORD=admin -p 9312:3000 --network=host grafana/grafana
2. run run.py in folder above user_data
3. in grafana install plugin simplejson
4. make new data source in grafana simplejson with url http://localhost:3004
5. place config for your strategy in user_data/configs
i think it's all its not complete but works maybe someone can pick it up from here
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": false,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1,
"links": [],
"liveNow": true,
"panels": [
{
"datasource": {
"type": "grafana-simple-json-datasource",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "percentage",
"steps": [
{
"color": "dark-red",
"value": null
},
{
"color": "green",
"value": 60
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 11,
"w": 22,
"x": 0,
"y": 0
},
"id": 6,
"options": {
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "/^Value$/",
"limit": 5000,
"values": true
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"text": {
"titleSize": 25,
"valueSize": 55
}
},
"pluginVersion": "9.4.7",
"targets": [
{
"datasource": {
"type": "grafana-simple-json-datasource",
"uid": "${datasource}"
},
"refId": "A",
"target": "backtest_results:$backtest",
"type": "timeserie"
}
],
"title": "Results",
"transformations": [
{
"id": "seriesToRows",
"options": {}
},
{
"id": "groupBy",
"options": {
"fields": {
"Metric": {
"aggregations": [],
"operation": "groupby"
},
"Time": {
"aggregations": [],
"operation": "aggregate"
},
"Value": {
"aggregations": [],
"operation": "groupby"
}
}
}
},
{
"id": "filterByValue",
"options": {
"filters": [
{
"config": {
"id": "equal",
"options": {
"value": "backtest_start_ts"
}
},
"fieldName": "Metric"
},
{
"config": {
"id": "equal",
"options": {
"value": "backtest_run_start_ts"
}
},
"fieldName": "Metric"
},
{
"config": {
"id": "equal",
"options": {
"value": "backtest_run_end_ts"
}
},
"fieldName": "Metric"
},
{
"config": {
"id": "greater",
"options": {
"value": 0
}
},
"fieldName": "Metric"
},
{
"config": {
"id": "equal",
"options": {
"value": "backtest_end_ts"
}
},
"fieldName": "Metric"
}
],
"match": "any",
"type": "exclude"
}
},
{
"id": "filterByValue",
"options": {
"filters": [
{
"config": {
"id": "equal",
"options": {
"value": "wins"
}
},
"fieldName": "Metric"
},
{
"config": {
"id": "equal",
"options": {
"value": "losses"
}
},
"fieldName": "Metric"
},
{
"config": {
"id": "equal",
"options": {
"value": "total_trades"
}
},
"fieldName": "Metric"
}
],
"match": "any",
"type": "include"
}
},
{
"id": "sortBy",
"options": {
"fields": {},
"sort": [
{
"desc": true,
"field": "Value"
}
]
}
}
],
"type": "gauge"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 11
},
"id": 4,
"panels": [],
"repeat": "backtest",
"repeatDirection": "h",
"title": "Backtest",
"type": "row"
},
{
"datasource": {
"type": "grafana-simple-json-datasource",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "right",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "stepBefore",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 6,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 21
}
]
},
"unit": "currencyUSD"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "volume"
},
"properties": [
{
"id": "custom.drawStyle",
"value": "bars"
},
{
"id": "custom.scaleDistribution",
"value": {
"linearThreshold": 99999999999,
"log": 10,
"type": "symlog"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "EMA50"
},
"properties": [
{
"id": "custom.drawStyle",
"value": "line"
}
]
},
{
"matcher": {
"id": "byName",
"options": "EMA200"
},
"properties": [
{
"id": "custom.drawStyle",
"value": "line"
}
]
},
{
"matcher": {
"id": "byName",
"options": "EMA800"
},
"properties": [
{
"id": "custom.drawStyle",
"value": "line"
}
]
},
{
"matcher": {
"id": "byName",
"options": "VWAP"
},
"properties": [
{
"id": "custom.drawStyle",
"value": "line"
},
{
"id": "custom.lineInterpolation",
"value": "linear"
},
{
"id": "custom.lineWidth",
"value": 2
}
]
},
{
"matcher": {
"id": "byName",
"options": "RSI"
},
"properties": [
{
"id": "custom.drawStyle",
"value": "line"
},
{
"id": "custom.scaleDistribution",
"value": {
"linearThreshold": 999999999999999,
"log": 2,
"type": "symlog"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "enter_long"
},
"properties": [
{
"id": "custom.drawStyle",
"value": "points"
},
{
"id": "custom.pointSize",
"value": 14
},
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "exit_long"
},
"properties": [
{
"id": "custom.drawStyle",
"value": "points"
},
{
"id": "color",
"value": {
"fixedColor": "blue",
"mode": "fixed"
}
},
{
"id": "custom.pointSize",
"value": 14
}
]
},
{
"matcher": {
"id": "byName",
"options": "averageVolume"
},
"properties": [
{
"id": "custom.drawStyle",
"value": "bars"
},
{
"id": "custom.scaleDistribution",
"value": {
"linearThreshold": 0,
"log": 2,
"type": "symlog"
}
}
]
},
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"open",
"high",
"low",
"close",
"daily_high",
"daily_low",
"phigh",
"VWAP",
"daily_open",
"weekly_high",
"weekly_low",
"prev_high",
"prev_low",
"enter_long"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
},
"gridPos": {
"h": 15,
"w": 24,
"x": 0,
"y": 12
},
"id": 2,
"options": {
"candleStyle": "candles",
"colorStrategy": "open-close",
"colors": {
"down": "red",
"up": "green"
},
"includeAllFields": true,
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"mode": "candles"
},
"repeat": "query0",
"repeatDirection": "h",
"targets": [
{
"datasource": {
"type": "grafana-simple-json-datasource",
"uid": "${datasource}"
},
"refId": "A",
"target": "candlesforstrategy:$strategy,$config,$pair,$timeframe",
"type": "timeserie"
}
],
"title": "Panel Title",
"transformations": [
{
"id": "filterFieldsByName",
"options": {
"include": {
"names": [
"Time",
"open",
"high",
"low",
"close",
"volume",
"daily_high",
"daily_low",
"phigh",
"VWAP",
"enter_long",
"weekly_high",
"weekly_low",
"prev_high",
"prev_low",
"daily_open"
]
}
}
}
],
"type": "candlestick"
}
],
"refresh": "",
"revision": 1,
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allValue": "ALL",
"current": {
"selected": false,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "grafana-simple-json-datasource",
"uid": "${datasource}"
},
"definition": "strategy_list:config.json",
"hide": 0,
"includeAll": true,
"label": "Strategy",
"multi": false,
"name": "strategy",
"options": [],
"query": "strategy_list:config.json",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
},
{
"current": {
"selected": false,
"text": "08:24:52 PM 04/13/23 NewDailyHigh backtest-result-2023-04-13_20-26-08.json",
"value": "08:24:52 PM 04/13/23 NewDailyHigh backtest-result-2023-04-13_20-26-08.json"
},
"datasource": {
"type": "grafana-simple-json-datasource",
"uid": "${datasource}"
},
"definition": "backtest_list:${strategy}",
"hide": 0,
"includeAll": false,
"label": "Backtest",
"multi": false,
"name": "backtest",
"options": [],
"query": "backtest_list:${strategy}",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
},
{
"current": {
"selected": false,
"text": "NewDailyHigh.config.json",
"value": "NewDailyHigh.config.json"
},
"datasource": {
"type": "grafana-simple-json-datasource",
"uid": "${datasource}"
},
"definition": "config_list",
"hide": 0,
"includeAll": false,
"label": "Config",
"multi": false,
"name": "config",
"options": [],
"query": "config_list",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
},
{
"current": {
"selected": false,
"text": "DOGE/USDT:USDT",
"value": "DOGE/USDT:USDT"
},
"datasource": {
"type": "grafana-simple-json-datasource",
"uid": "${datasource}"
},
"definition": "pairlist:$strategy,$config",
"hide": 0,
"includeAll": false,
"label": "Pair",
"multi": false,
"name": "pair",
"options": [],
"query": "pairlist:$strategy,$config",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
},
{
"current": {
"selected": false,
"text": "15m",
"value": "15m"
},
"hide": 0,
"includeAll": false,
"label": "Timeframe",
"multi": false,
"name": "timeframe",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": false,
"text": "5m",
"value": "5m"
},
{
"selected": true,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
},
{
"selected": false,
"text": "1d",
"value": "1d"
}
],
"query": "1m,5m,15m,1h,1d",
"queryValue": "",
"skipUrlSync": false,
"type": "custom"
},
{
"current": {
"selected": false,
"text": "run.py",
"value": "run.py"
},
"hide": 0,
"includeAll": false,
"label": "DataSource",
"multi": false,
"name": "datasource",
"options": [],
"query": "grafana-simple-json-datasource",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "2023-04-03T10:45:17.668Z",
"to": "2023-04-03T22:41:52.000Z"
},
"timepicker": {
"refresh_intervals": [
"10ms",
"100ms",
"250ms",
"500ms",
"1s",
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
},
"timezone": "",
"title": "New dashboard",
"uid": "bVIAZ3LVz",
"version": 27,
"weekStart": ""
}
import logging
from datetime import datetime
from pathlib import Path
import pandas as pd
from freqtrade.configuration import Configuration
from freqtrade.data import btanalysis
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history import load_pair_history
from freqtrade.enums import CandleType
from freqtrade.resolvers import StrategyResolver
from grafana_pandas_datasource import create_app
from grafana_pandas_datasource.registry import data_generators as dg
from grafana_pandas_datasource.service import pandas_component
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
TZONE = datetime.now().astimezone().tzinfo
def get_variable_strategy_list(config_list=None):
"""
Returns a list of all available strategies
as will be listed in the strategy dropdown.
"""
from freqtrade.resolvers import StrategyResolver
from freqtrade.configuration import Configuration
strategy_list = StrategyResolver.search_all_objects(
config=Configuration.from_files(
files=["config.json"],
),
enum_failed=False,
)
return [s["name"] for s in strategy_list]
dg.add_metric_finder("strategy_list", get_variable_strategy_list)
def get_variable_backtest_list(strategy_name=None):
"""
Returns a list of all available backtests for a strategy.
"""
backtest_list = btanalysis.get_backtest_resultlist(
dirname=Path("user_data/backtest_results"),
)
# filter by strategy name
if strategy_name and strategy_name.lower() != "all":
backtest_list = [bt for bt in backtest_list if bt["strategy"] == strategy_name]
# format backtest name
bname = (
lambda bt: f"{datetime.fromtimestamp(bt['backtest_start_time']).strftime('%r %D')} {bt['strategy']} {bt['filename']}"
)
return [bname(bt) for bt in backtest_list]
dg.add_metric_finder("backtest_list", get_variable_backtest_list)
def get_backtest_results(backtest_name=None, ts_range=None):
"""
Returns a dataframe with backtest results.
"""
# get backtest name based of formatting from get_variable_backtest_list
spl = list(map(lambda s: s.replace("""\\""", ""), backtest_name.split(" ")))
strategy_name = spl[3]
backtest_name = spl[-1]
# get backtest list
backtest_list = btanalysis.get_backtest_resultlist(
dirname=Path("user_data/backtest_results"),
)
# filter by backtest name
results = [bt for bt in backtest_list if bt["filename"] == backtest_name]
result = results[0]
stats = btanalysis.load_backtest_stats(
filename=f"user_data/backtest_results/{backtest_name}",
)
stats = stats['strategy'][strategy_name]
#
# trades = btanalysis.load_trades(
# db_url="",
# source='file',
# strategy=strategy_name,
# exportfilename=Path(f"user_data/backtest_results/{backtest_name}"),
# )
stats.pop('trades')
data = pd.json_normalize(stats)
data['date'] = datetime.fromtimestamp(result["backtest_start_time"], tz=TZONE)
data = data.set_index('date')
return data
dg.add_metric_reader("backtest_results", get_backtest_results)
def get_config_list(*args):
"""
Returns a list of all available config files
from directory `user_data/configs`
"""
config_list = list(Path("user_data/configs").glob("*.json"))
return [c.name for c in config_list]
dg.add_metric_finder("config_list", get_config_list)
def get_pairlist(query=None):
spl = query.split(',')
strategy, config = spl[0], spl[1]
config = config.replace("""\\""", "")
config = Configuration.from_files(["user_data/configs/" + config])
config['strategy'] = strategy
return config['exchange']['pair_whitelist']
dg.add_metric_finder("pairlist", get_pairlist)
# todo add caching
def get_candles_for_strategy(query=None, ts_range=None):
"""
Returns a dataframe with candles.
"""
spl = query.split(',')
strategy, config, pair, timeframe = spl[0], spl[1], spl[2], spl[3]
config = config.replace("""\\""", "")
pair = pair.replace("""\\""", "")
strategy = strategy.replace("""\\""", "")
config = Configuration.from_files(["user_data/configs/" + config])
config["strategy"] = strategy
config["timeframe"] = timeframe
config["exchange"]["pair_whitelist"] = [pair]
# todo add timerange
df = load_pair_history(
datadir=config["datadir"],
timeframe=timeframe,
pair=pair,
data_format="json",
candle_type=getattr(CandleType, config["trading_mode"].name),
)
time_filter = (
(df['date'] > ts_range["$gt"]) &
(df['date'] < ts_range["$lte"])
)
s = StrategyResolver.load_strategy(config)
s.dp = DataProvider(config, None, None)
s.ft_bot_start()
try:
df = s.analyze_ticker(df, {'pair': pair})
except Exception as e:
df['error'] = True
df.set_index("date", inplace=True)
df["date"] = df.index
logger.debug(e)
return df
# check if we have key enter_long
if 'enter_long' in df:
df.loc[(
df['enter_long'] == 1
), ['enter_long']] = df['close']
# check if we have key exit_long
if 'exit_long' in df:
df.loc[(
df['exit_long'] == 1
), ['exit_long']] = df['close']
df = df[time_filter]
df.set_index("date", inplace=True)
df["date"] = df.index
return df
dg.add_metric_reader("candlesforstrategy", get_candles_for_strategy)
def main():
logger.debug("22222????????")
# Define and register data generators.
# Create Flask application.
app = create_app()
# Register pandas component.
app.register_blueprint(pandas_component, url_prefix="/")
app.run(host="0.0.0.0", port=3004, debug=True, use_reloader=False)
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment