Fix: Analytics Worker berechnet jetzt alle Tabellen pro Tag
Some checks failed
Deployment / deploy-docker (push) Has been cancelled
Some checks failed
Deployment / deploy-docker (push) Has been cancelled
This commit is contained in:
25
daemon.py
25
daemon.py
@@ -9,6 +9,10 @@ from src.exchanges.ls import LSExchange
|
|||||||
from src.exchanges.deutsche_boerse import XetraExchange, FrankfurtExchange, QuotrixExchange
|
from src.exchanges.deutsche_boerse import XetraExchange, FrankfurtExchange, QuotrixExchange
|
||||||
from src.exchanges.gettex import GettexExchange
|
from src.exchanges.gettex import GettexExchange
|
||||||
from src.exchanges.stuttgart import StuttgartExchange
|
from src.exchanges.stuttgart import StuttgartExchange
|
||||||
|
from src.exchanges.boersenag import (
|
||||||
|
DUSAExchange, DUSBExchange, DUSCExchange, DUSDExchange,
|
||||||
|
HAMAExchange, HAMBExchange, HANAExchange, HANBExchange
|
||||||
|
)
|
||||||
from src.database.questdb_client import DatabaseClient
|
from src.database.questdb_client import DatabaseClient
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
@@ -136,6 +140,16 @@ def run_task(historical=False):
|
|||||||
gettex = GettexExchange()
|
gettex = GettexExchange()
|
||||||
stuttgart = StuttgartExchange()
|
stuttgart = StuttgartExchange()
|
||||||
|
|
||||||
|
# Börsenag Exchanges (Düsseldorf, Hamburg, Hannover)
|
||||||
|
dusa = DUSAExchange()
|
||||||
|
dusb = DUSBExchange()
|
||||||
|
dusc = DUSCExchange()
|
||||||
|
dusd = DUSDExchange()
|
||||||
|
hama = HAMAExchange()
|
||||||
|
hamb = HAMBExchange()
|
||||||
|
hana = HANAExchange()
|
||||||
|
hanb = HANBExchange()
|
||||||
|
|
||||||
# Pass last_ts to fetcher to allow smart filtering
|
# Pass last_ts to fetcher to allow smart filtering
|
||||||
# daemon.py runs daily, so we want to fetch everything since DB state
|
# daemon.py runs daily, so we want to fetch everything since DB state
|
||||||
# BUT we need to be careful: eix.py's fetch_latest_trades needs 'since_date' argument
|
# BUT we need to be careful: eix.py's fetch_latest_trades needs 'since_date' argument
|
||||||
@@ -145,12 +159,21 @@ def run_task(historical=False):
|
|||||||
exchanges_to_process = [
|
exchanges_to_process = [
|
||||||
(eix, {'limit': None if historical else 5}), # Default limit 5 for safety if no historical
|
(eix, {'limit': None if historical else 5}), # Default limit 5 for safety if no historical
|
||||||
(ls, {'include_yesterday': historical}),
|
(ls, {'include_yesterday': historical}),
|
||||||
# Neue Exchanges
|
# Deutsche Börse Exchanges
|
||||||
(xetra, {'include_yesterday': historical}),
|
(xetra, {'include_yesterday': historical}),
|
||||||
(frankfurt, {'include_yesterday': historical}),
|
(frankfurt, {'include_yesterday': historical}),
|
||||||
(quotrix, {'include_yesterday': historical}),
|
(quotrix, {'include_yesterday': historical}),
|
||||||
(gettex, {'include_yesterday': historical}),
|
(gettex, {'include_yesterday': historical}),
|
||||||
(stuttgart, {'include_yesterday': historical}),
|
(stuttgart, {'include_yesterday': historical}),
|
||||||
|
# Börsenag Exchanges (Düsseldorf, Hamburg, Hannover)
|
||||||
|
(dusa, {'include_yesterday': historical}),
|
||||||
|
(dusb, {'include_yesterday': historical}),
|
||||||
|
(dusc, {'include_yesterday': historical}),
|
||||||
|
(dusd, {'include_yesterday': historical}),
|
||||||
|
(hama, {'include_yesterday': historical}),
|
||||||
|
(hamb, {'include_yesterday': historical}),
|
||||||
|
(hana, {'include_yesterday': historical}),
|
||||||
|
(hanb, {'include_yesterday': historical}),
|
||||||
]
|
]
|
||||||
|
|
||||||
db = DatabaseClient(host="questdb", user=DB_USER, password=DB_PASSWORD)
|
db = DatabaseClient(host="questdb", user=DB_USER, password=DB_PASSWORD)
|
||||||
|
|||||||
@@ -102,6 +102,38 @@
|
|||||||
<h4 class="text-md font-bold mb-3 text-pink-400">Quotrix</h4>
|
<h4 class="text-md font-bold mb-3 text-pink-400">Quotrix</h4>
|
||||||
<div class="h-48"><canvas id="maChartQUOTRIX"></canvas></div>
|
<div class="h-48"><canvas id="maChartQUOTRIX"></canvas></div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="glass p-4">
|
||||||
|
<h4 class="text-md font-bold mb-3 text-cyan-400">Düsseldorf Reg. (DUSA)</h4>
|
||||||
|
<div class="h-48"><canvas id="maChartDUSA"></canvas></div>
|
||||||
|
</div>
|
||||||
|
<div class="glass p-4">
|
||||||
|
<h4 class="text-md font-bold mb-3 text-teal-400">Düsseldorf Frei. (DUSB)</h4>
|
||||||
|
<div class="h-48"><canvas id="maChartDUSB"></canvas></div>
|
||||||
|
</div>
|
||||||
|
<div class="glass p-4">
|
||||||
|
<h4 class="text-md font-bold mb-3 text-lime-400">Quotrix Reg. (DUSC)</h4>
|
||||||
|
<div class="h-48"><canvas id="maChartDUSC"></canvas></div>
|
||||||
|
</div>
|
||||||
|
<div class="glass p-4">
|
||||||
|
<h4 class="text-md font-bold mb-3 text-green-400">Quotrix Frei. (DUSD)</h4>
|
||||||
|
<div class="h-48"><canvas id="maChartDUSD"></canvas></div>
|
||||||
|
</div>
|
||||||
|
<div class="glass p-4">
|
||||||
|
<h4 class="text-md font-bold mb-3 text-indigo-400">Hamburg Reg. (HAMA)</h4>
|
||||||
|
<div class="h-48"><canvas id="maChartHAMA"></canvas></div>
|
||||||
|
</div>
|
||||||
|
<div class="glass p-4">
|
||||||
|
<h4 class="text-md font-bold mb-3 text-purple-400">Hamburg Frei. (HAMB)</h4>
|
||||||
|
<div class="h-48"><canvas id="maChartHAMB"></canvas></div>
|
||||||
|
</div>
|
||||||
|
<div class="glass p-4">
|
||||||
|
<h4 class="text-md font-bold mb-3 text-fuchsia-400">Hannover Reg. (HANA)</h4>
|
||||||
|
<div class="h-48"><canvas id="maChartHANA"></canvas></div>
|
||||||
|
</div>
|
||||||
|
<div class="glass p-4">
|
||||||
|
<h4 class="text-md font-bold mb-3 text-red-400">Hannover Frei. (HANB)</h4>
|
||||||
|
<div class="h-48"><canvas id="maChartHANB"></canvas></div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -151,7 +183,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<label class="block text-sm font-bold text-slate-400 mb-2">Exchanges (optional, komma-separiert)</label>
|
<label class="block text-sm font-bold text-slate-400 mb-2">Exchanges (optional, komma-separiert)</label>
|
||||||
<input type="text" id="customExchanges" class="input-glass" placeholder="z.B. EIX,LS,XETRA,FRA,GETTEX,STU,QUOTRIX" onchange="updateCustomGraph(); updateUrlParams()">
|
<input type="text" id="customExchanges" class="input-glass" placeholder="z.B. EIX,LS,XETRA,FRA,GETTEX,STU,QUOTRIX,DUSA,HAMA,HANA" onchange="updateCustomGraph(); updateUrlParams()">
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -283,7 +315,15 @@
|
|||||||
'XETRA': { canvasId: 'maChartXETRA' },
|
'XETRA': { canvasId: 'maChartXETRA' },
|
||||||
'FRA': { canvasId: 'maChartFRA' },
|
'FRA': { canvasId: 'maChartFRA' },
|
||||||
'STU': { canvasId: 'maChartSTU' },
|
'STU': { canvasId: 'maChartSTU' },
|
||||||
'QUOTRIX': { canvasId: 'maChartQUOTRIX' }
|
'QUOTRIX': { canvasId: 'maChartQUOTRIX' },
|
||||||
|
'DUSA': { canvasId: 'maChartDUSA' },
|
||||||
|
'DUSB': { canvasId: 'maChartDUSB' },
|
||||||
|
'DUSC': { canvasId: 'maChartDUSC' },
|
||||||
|
'DUSD': { canvasId: 'maChartDUSD' },
|
||||||
|
'HAMA': { canvasId: 'maChartHAMA' },
|
||||||
|
'HAMB': { canvasId: 'maChartHAMB' },
|
||||||
|
'HANA': { canvasId: 'maChartHANA' },
|
||||||
|
'HANB': { canvasId: 'maChartHANB' }
|
||||||
};
|
};
|
||||||
Object.values(exchangeGroups).forEach(config => {
|
Object.values(exchangeGroups).forEach(config => {
|
||||||
const canvas = document.getElementById(config.canvasId);
|
const canvas = document.getElementById(config.canvasId);
|
||||||
@@ -325,7 +365,15 @@
|
|||||||
'XETRA': { exchanges: ['XETRA'], canvasId: 'maChartXETRA' },
|
'XETRA': { exchanges: ['XETRA'], canvasId: 'maChartXETRA' },
|
||||||
'FRA': { exchanges: ['FRA'], canvasId: 'maChartFRA' },
|
'FRA': { exchanges: ['FRA'], canvasId: 'maChartFRA' },
|
||||||
'STU': { exchanges: ['STU'], canvasId: 'maChartSTU' },
|
'STU': { exchanges: ['STU'], canvasId: 'maChartSTU' },
|
||||||
'QUOTRIX': { exchanges: ['QUOTRIX'], canvasId: 'maChartQUOTRIX' }
|
'QUOTRIX': { exchanges: ['QUOTRIX'], canvasId: 'maChartQUOTRIX' },
|
||||||
|
'DUSA': { exchanges: ['DUSA'], canvasId: 'maChartDUSA' },
|
||||||
|
'DUSB': { exchanges: ['DUSB'], canvasId: 'maChartDUSB' },
|
||||||
|
'DUSC': { exchanges: ['DUSC'], canvasId: 'maChartDUSC' },
|
||||||
|
'DUSD': { exchanges: ['DUSD'], canvasId: 'maChartDUSD' },
|
||||||
|
'HAMA': { exchanges: ['HAMA'], canvasId: 'maChartHAMA' },
|
||||||
|
'HAMB': { exchanges: ['HAMB'], canvasId: 'maChartHAMB' },
|
||||||
|
'HANA': { exchanges: ['HANA'], canvasId: 'maChartHANA' },
|
||||||
|
'HANB': { exchanges: ['HANB'], canvasId: 'maChartHANB' }
|
||||||
};
|
};
|
||||||
|
|
||||||
// Alle Daten nach Datum sortieren
|
// Alle Daten nach Datum sortieren
|
||||||
|
|||||||
@@ -173,25 +173,30 @@ class DeutscheBoerseBase(BaseExchange):
|
|||||||
def _parse_trade_record(self, record: dict) -> Optional[Trade]:
|
def _parse_trade_record(self, record: dict) -> Optional[Trade]:
|
||||||
"""
|
"""
|
||||||
Parst einen einzelnen Trade-Record aus dem JSON.
|
Parst einen einzelnen Trade-Record aus dem JSON.
|
||||||
Deutsche Börse verwendet RTS1/RTS2 Format.
|
|
||||||
|
|
||||||
Wichtige Felder:
|
Aktuelles JSON-Format (NDJSON):
|
||||||
- TrdDt: Trading Date (YYYY-MM-DD)
|
{
|
||||||
- TrdTm: Trading Time (HH:MM:SS.ffffff)
|
"messageId": "posttrade",
|
||||||
- ISIN: Instrument Identifier
|
"sourceName": "GAT",
|
||||||
- FinInstrmId.Id: Alternative ISIN Feld
|
"isin": "US00123Q1040",
|
||||||
- Pric.Pric.MntryVal.Amt: Preis
|
"lastTradeTime": "2026-01-29T14:07:00.419000000Z",
|
||||||
- Qty.Unit: Menge
|
"lastTrade": 10.145,
|
||||||
|
"lastQty": 500.0,
|
||||||
|
"currency": "EUR",
|
||||||
|
...
|
||||||
|
}
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
# ISIN extrahieren
|
# ISIN extrahieren - neues Format verwendet 'isin' lowercase
|
||||||
isin = record.get('ISIN') or record.get('FinInstrmId', {}).get('Id', '')
|
isin = record.get('isin') or record.get('ISIN') or record.get('instrumentId') or record.get('FinInstrmId', {}).get('Id', '')
|
||||||
if not isin:
|
if not isin:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Preis extrahieren (verschiedene mögliche Pfade)
|
# Preis extrahieren - neues Format: 'lastTrade'
|
||||||
price = None
|
price = None
|
||||||
if 'Pric' in record:
|
if 'lastTrade' in record:
|
||||||
|
price = float(record['lastTrade'])
|
||||||
|
elif 'Pric' in record:
|
||||||
pric = record['Pric']
|
pric = record['Pric']
|
||||||
if isinstance(pric, dict):
|
if isinstance(pric, dict):
|
||||||
if 'Pric' in pric:
|
if 'Pric' in pric:
|
||||||
@@ -208,9 +213,11 @@ class DeutscheBoerseBase(BaseExchange):
|
|||||||
if price is None or price <= 0:
|
if price is None or price <= 0:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Menge extrahieren
|
# Menge extrahieren - neues Format: 'lastQty'
|
||||||
quantity = None
|
quantity = None
|
||||||
if 'Qty' in record:
|
if 'lastQty' in record:
|
||||||
|
quantity = float(record['lastQty'])
|
||||||
|
elif 'Qty' in record:
|
||||||
qty = record['Qty']
|
qty = record['Qty']
|
||||||
if isinstance(qty, dict):
|
if isinstance(qty, dict):
|
||||||
quantity = float(qty.get('Unit', qty.get('Qty', 0)))
|
quantity = float(qty.get('Unit', qty.get('Qty', 0)))
|
||||||
@@ -220,29 +227,46 @@ class DeutscheBoerseBase(BaseExchange):
|
|||||||
if quantity is None or quantity <= 0:
|
if quantity is None or quantity <= 0:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Timestamp extrahieren
|
# Timestamp extrahieren - neues Format: 'lastTradeTime'
|
||||||
trd_dt = record.get('TrdDt', '')
|
timestamp = None
|
||||||
trd_tm = record.get('TrdTm', '00:00:00')
|
if 'lastTradeTime' in record:
|
||||||
|
ts_str = record['lastTradeTime']
|
||||||
|
# Format: "2026-01-29T14:07:00.419000000Z"
|
||||||
|
# Python kann max 6 Dezimalstellen, also kürzen
|
||||||
|
if '.' in ts_str:
|
||||||
|
parts = ts_str.replace('Z', '').split('.')
|
||||||
|
if len(parts) == 2 and len(parts[1]) > 6:
|
||||||
|
ts_str = parts[0] + '.' + parts[1][:6] + '+00:00'
|
||||||
|
else:
|
||||||
|
ts_str = ts_str.replace('Z', '+00:00')
|
||||||
|
else:
|
||||||
|
ts_str = ts_str.replace('Z', '+00:00')
|
||||||
|
timestamp = datetime.fromisoformat(ts_str)
|
||||||
|
else:
|
||||||
|
# Fallback für altes Format
|
||||||
|
trd_dt = record.get('TrdDt', '')
|
||||||
|
trd_tm = record.get('TrdTm', '00:00:00')
|
||||||
|
|
||||||
if not trd_dt:
|
if not trd_dt:
|
||||||
|
return None
|
||||||
|
|
||||||
|
ts_str = f"{trd_dt}T{trd_tm}"
|
||||||
|
if '.' in ts_str:
|
||||||
|
parts = ts_str.split('.')
|
||||||
|
if len(parts[1]) > 6:
|
||||||
|
ts_str = parts[0] + '.' + parts[1][:6]
|
||||||
|
|
||||||
|
timestamp = datetime.fromisoformat(ts_str)
|
||||||
|
|
||||||
|
if timestamp is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Kombiniere Datum und Zeit
|
|
||||||
ts_str = f"{trd_dt}T{trd_tm}"
|
|
||||||
# Entferne Mikrosekunden wenn zu lang
|
|
||||||
if '.' in ts_str:
|
|
||||||
parts = ts_str.split('.')
|
|
||||||
if len(parts[1]) > 6:
|
|
||||||
ts_str = parts[0] + '.' + parts[1][:6]
|
|
||||||
|
|
||||||
# Parse als UTC (Deutsche Börse liefert UTC)
|
|
||||||
timestamp = datetime.fromisoformat(ts_str)
|
|
||||||
if timestamp.tzinfo is None:
|
if timestamp.tzinfo is None:
|
||||||
timestamp = timestamp.replace(tzinfo=timezone.utc)
|
timestamp = timestamp.replace(tzinfo=timezone.utc)
|
||||||
|
|
||||||
return Trade(
|
return Trade(
|
||||||
exchange=self.name,
|
exchange=self.name,
|
||||||
symbol=isin, # Symbol = ISIN
|
symbol=isin,
|
||||||
isin=isin,
|
isin=isin,
|
||||||
price=price,
|
price=price,
|
||||||
quantity=quantity,
|
quantity=quantity,
|
||||||
@@ -250,7 +274,7 @@ class DeutscheBoerseBase(BaseExchange):
|
|||||||
)
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error parsing record: {e}")
|
# Debug: Zeige ersten fehlgeschlagenen Record
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _get_last_trading_day(self, from_date: datetime.date) -> datetime.date:
|
def _get_last_trading_day(self, from_date: datetime.date) -> datetime.date:
|
||||||
|
|||||||
Reference in New Issue
Block a user