From 228212cbabc529e75268b352f2afe3034e5afc1a Mon Sep 17 00:00:00 2001 From: Melchior Reimers Date: Tue, 27 Jan 2026 11:09:52 +0100 Subject: [PATCH] updated dashboard --- .../deutsche_boerse.cpython-313.pyc | Bin 13896 -> 14960 bytes .../__pycache__/gettex.cpython-313.pyc | Bin 18788 -> 19021 bytes src/exchanges/deutsche_boerse.py | 113 +++++++++++------- src/exchanges/gettex.py | 11 +- 4 files changed, 80 insertions(+), 44 deletions(-) diff --git a/src/exchanges/__pycache__/deutsche_boerse.cpython-313.pyc b/src/exchanges/__pycache__/deutsche_boerse.cpython-313.pyc index 034c26f630954f63abfaa1e5ec5103303bc56446..867d3a0154a9102db8a9abbafd924f1d5ce176cb 100644 GIT binary patch delta 4834 zcmbtY32>9g72aR>VI4+?Wl6Sw8DGNpiE&*(!3WsbfP@HOh(_45Q85x($pc}MicH(o zNi!{G+hUhACTXWQX%17jDQVKyDQV&yo!B+OMsW$9(#bGwXUssGY3NM*-Y;Yu544%- z%Jc2px4UoOvB(K{d2-q2`N1WNNaj|3|!2DBFLuLJo7F|JiYB_84twMtv0RN5k= zvKBd&^8|R}Rz-`FDqB=k1#O8(+Ny3zp(!mIsuB8St=bkH)d4N1dXIvpdX#j9N99p_ zQaqXxF*SI!)acQ9^qy4D3TT@wB%62)R^l=4NvRg2HZ6ouv%oe1+q}n|U;{Tz;HClB zvV?09xaq*PF5#y4a84VOaNUL;6iUdIb=Ra&UBfll(_H@80QZ@QX|4^RiYR7(|`m;zxHS8Hpby_t{aH^>d z=hY(ABe>Y#G+vmsLEFT+*b~}+c`$Zp&5f79utljniA!@?I&nwCyYPN4hw&}$$inB;J=UA#o6&qTS5&ISdt|lo?E`*pT zS)}b0A>CwQmMl5KjazeBan~(lKocbW0Njw`O>yHsURK;GNe26;MSXh57&7w9Jxb+@7VqjUjL8V@{f!Sq)tWoDpfn&C0TP9^D+@6r%&E4VKRp^oB z$6>V|53J{N%1g$?naT{tt?eL@1iuZ3I`|?0#rull{yJ*e%e(#20N)!t7>owGiul9+ zU^E!&;g3opB_$t&0AazzWZFB zMxWQcIWDFCP){JPZgcJQxf{2;H#K@_DK3_aPzDfJN22~{U&PlL?h3@sZrAok-{wti zo3;z%cljC`H@kLG+`iO-!zIz+!9ZLZ=?w%9(Q<*)b2NA;&TOW7u7d3}@o5{7t84LW zX>SOi)J96flvcB+O|}?rS*Ip(TW-a1vMBX;1tM`pcVBO>uaEYE&VeX>h~mh&63zDs zOLE=Rk1R=VFcgT(D2S$Aaj7t!u7RG227q5$C|)dai?1tuIMf^VclrD*$6Ocd!oj_8 zO&lXvQVspGIdlG`sbGW~DHyR%8mjsgS54NZzA$6T?{AzJRcXv~ww1&6->V-{yq-T% zF=cB%uDD{b%^32h4EZCK(}t2Owu}>-?qpr%YtH44J6?58@eRi}%^A}NU4vc2!J*(_ z$7HNv{doGgdZKF5y7hu_+nmvIvgJg};6oRT&Pzu76}@Ff@0`**FX#(z%81!|a_@<~ zFI2o-_k7*x{+XJlshXzg!sg$bHcq%^GqZ-fhq_Pg9}W+NM|!3+YmPU6DkkPl+`Nof zvQ9oS@W{#7Kuk~%I2k#^Z9{FRw!~)acXq(JgKW-UUO=M4Oo7gaE-g-g54f?O>O#PZ$^GX=~S>!R;dC$fQ zn;oJz^e!z_zu{dXsLp5eO-|{XqD-LQT)77Dw;Ai2>LhR3WkA2>R5h(uzEvqly3W{~ zDSull2Kwz(CDK-b&ft*F6lh0gt5N#)I+vv-L;6l;E#SKu8eqM+cLu zDYnFY(N2mPh;Ttr%!h>g!l_Lh^9Pafb%cI^xa1HGcE+W_P$)p@JqnzlTv}dqH-Y#KD#OO}3d#*$1#}wkG09Z9 z;8{>ioI{*ucD~4%5FKGF+*Iw-yiWuqm&9_F~)ImD12Dhz4SOPg`0(bA7SC4qnI4gc7!T` zxH1sxI~bs_hiC-bvcQpOAk;-Mho^gxqhIoN3a_T{&cUlrhjGkl9Ha8XBbU%o_)dsx z!Zg?u4EcL~xIJktdwx|$Nn9NbM}Z}n73YGq6$KmsxZ4}Y{p3Me%Guiea2KVWvUphnl{z;H(l15*s&s0_i+_G4*Jy7 zsRQe#b=h-z<4M&C)dgGO1$_~8q^>yb8jube9)ILC_o2>l$BlP-^JwbmwzH;D`RIe^ zZ09uNJKyh|bnkgDJlXcZw+_Bp&sOHxsTQZ*RhEO5l%@4FV$_qf%n5~Xc>8lI zdKTyY1Yr!}rwBhocm?6-2#bMu6`RZa^&0XJ*0LAM8fAhbUju9RGhewOy9Ji+OM~9~ z&c8w<0lo<`%>&(iG|CtA4NYzrO*rGDa;^EU24t{*muIQlKnnd8E_B;s9nyE5;9$WD zTkW=`78{pXOy5|Txft^E$U|shuUFhQ;k;l%xH6U7!5*zF6@uoymyY(T4E3%%b&OH{ z!b-~{?0}xQaa&&)cHxnQ7>E1^1ACY1ZTVk%zk@3v?7(UDKs-h+Ny@HBHS;3&NmZpo zFcAkht%-boFV3hy@F8@tn(B42Gf3`6fIFU0d<^Iggfax-JP1J++$5|icz(fuaYg;X za93Y%U@e`38K4V^oC1I>jpIHdx{ruj_)`LYOe&^G#mB_)G0FV{$$*=~s7sq9`q^~r kBr(rwb;s1~a&@`3L&8}H?Kk0n8=Up#!z`mFUxMrX2bJSA`~Uy| delta 3757 zcma)8du*Fm6~CV!pWpuM_!&Q1+ewqSdCqxvY16bx7ThKs_RO}XTjbVG>ov8v>y&n( z!re4bCWMysGLkY-tEpqz25MP=1;(WA9l;;Oq%4`&(SinO0wGt8sGuNm&gVKwrw;fM z{qFI(_ndQo=RVHOkIs+iuj_Pb0^j@JIG(s%GpV<8%afH=8ObhLP)21zIh6-_%6mwd zgr&O_K_yiNRa6DPWnuX)bx=b!K`qsa{roOnP*3$BE5gdKDy*)PQX#CNhOjoQ3+uxI z{5IN&kAw{l5;pGBG)pluA+$u)mVnl@uLQL%qS_>?&7igv)S}ibYOSEPtC)XjGCFIqQ( ziaHV2o19WqaTUz3sNmeQ_bSRHd>M`?XOF9D>}Zjbj>MBWRnPGQ@kd7!sia>{OW7N$ zD$c?Fq-x=+nOprf*E0Ki^&cgYPo->L*KBJ9GrxwGq9Y_sw;*h1W_<*#9?|!5ce5Yq z|7B5u8Z&kMt1?2KCG{jJ*+AGu!KRde7C-g`Z*llYL^{Y96oiB&VQITGDT~NLO4cPa z8D(1@AS4PF^`xADR&%P#0c$m=En0;1>|}{qHE6h{U`Dfr2dA=-nLTWDYUJ&lge2t= zdB_H)l29qS-N-j%PT@=PuHd}~r3az7t(!btX4qe@eZTTn?tZZx(ii^e9>w+~z> zB8pWLcDO{%?lyV^MR8D486lxE_DzjzUs4rORTbirsELq>B(&+)&vi{iRW$XGe(>M_ zFd-V^fzhkBm4pmd7fr${2vEbGQCce0gY`viL{jvkCt*#5)04&`X(l)MESI)QH4{)%k<0(-=A?>5X(;#;tb{9H~koEV9x_CGQvKRP;G+cuEn)U*cr$5an_?v2uP zJmwkGc}8h87EgPIY3hiFDXrTA&YZD(&;0|t_jGj+?dS^k3`Ih_X)V}M@s!qql+&a~ zqoYUDLkCi^IIU-|TYK-p4Cmw{sc0;nGYvh8{)a^0Lo|LMMPqaa4iJLjJpZgc7|_r=4!hed6cc8aPTH%teym5n5l@f=@poRf=TXe;!jVOJ2( z+6TAO+qPf^wkk>g8M1PcTs4};`Mk`p8sI*0R45{f`g(oG}DfcP&Ir`$* z^J5pPmI6DnfgOvLJAY&Bp68Yw&NCIKE1t_#ovA-vKU2Br2uvt$NJ&W#m*s04tr|1_nrJggo)4Jz`C3|hwUON-}z}`3^Tedl;#wN$6PE4No(Co@flt%YuxA)B6 z@9dqCExKzbdY4_EC0A3{)pX%Yi>{7|p5NJA%k~ZE>1_N1d(AZ|$=Kc3bfmQD?+#+H zUN%^!x#{vV{?q&Xyg0UC4P7#X^9C36>%iE0nGEEqD+ zj6FTJBy7qGn`TUlLd|VcSSa5*cW7?!MLO@DH!WEEFBt}I=9S_EZmuYZp%lWvVqMl6 zC%PtLQ;EsM#NdLbWA1^u;Klv(#(C*N$?hd>|AMyvs$dz{-OMX+_&?LfV9lRu>D^3T z@5)G^d2_3$r&{`!djrUCb+iM%-O}09A$g~{3FLRH)jjQscbgR`cbI!C(s^D7^1Nw7U#oP!RSq&uK=1>rx6iE0d70QSj>vbqu}G48lo(oa|AWIFOSklF2xwPq3$bRb@|N zr?~O;Ag%3tl&AwcqwJEentSHevUAd&PvxjlB6J{p1z{V2Uy@TCi^mVgqK{K3i^MN2 zOkfh^oGca{ji<3@icZ<572Vt)SY72_cn~~Y*`UD}k62Mhq_gi-&PXy51ymp)19gRj zu=aoETdO2^(H8IY7^JJ;4b?{)0{rY3Q5Wk5R2*ekQmX_1%Kqj;4H`5S$A@JR*16fP z&_|STBMab&QG$97j7$6a6;A+U3=TKl>4|w9vx%WWmeM6l zAZrO+Xj-(ijrU%GbMa;7ZLp|LsjsPs#k-kEgeU?gAt&3PN{!Ix*<*J(OV6Pe*RZeyOHe{^uov!{w$;n67fg8q z&765I{xjzaiuwKQzCdero!oKZzC3|u{*c%#>>pk@IFj9e~2 zycK&bSLbmIf|tGA)WeG%=Yi@W_E58lt7VThdv4MFu=#W8{=0djrU9l&Uqx@Xn8JDa zZ%nPdG@3j-e3Xvf#JVzt46m54lMiLyt8yLtWlM8`&1)i? zBdtcRon>0~84(Fa+3duPUnVgv!7OyUU;msZS>7@)B8 zL`Qg6=ad~sQn8~W@on^7_zBTT`fC7CUODa~B78(N;=k$&31ms&3h`bcn?5FPsFwO2 p+_I)~S=f0^A=64uD)VZ>Tw9#_ZW-s8cC8R>R|Z(mmP#3h`w!7GI=TP= diff --git a/src/exchanges/__pycache__/gettex.cpython-313.pyc b/src/exchanges/__pycache__/gettex.cpython-313.pyc index 651d9914d5ebacad9c61ad7d94034710c4592b1e..92748eda8819af8b3e2dec2ebba0f2abe4144e7f 100644 GIT binary patch delta 951 zcma)4O-vI(6n?MmwzO<%w_R*YsaB-g)2$jtLtu{p%p;U#T`^|+tp9?6Ihv8pYM|o#k3Vym>nXA z0ZwZGMt&*~-$2i9+nlf}R{u&FdIh#x8~6R4{sW4&py36@lJFEttJ<=rid}P)4d4jV z?}`E>Hqx{0<<1I71PjYoMY@tSyDf@U)LidDfu5@O(ZL3uu`>Q*$d%pc z&OsjMkF8+3?`XPzaG`JL+j2VBL-$4eSV|`%r8rBcBdz$5K8n;dzGq_jAC@}AhMX1= xGhoR;aRy8oFfx*b+6Ac1!ul-uUV$g42houRD-A{i!d?TexzO_s@<`lF;14kin`8Tly#SIJ@xk=N~Y zUZ>^^8zyLsnGi}Jqf9edl)Qp71_yW$rQ>8Io`eu(dB{qf0~c9~pP_j#-r#g=E~*@* z$_}dBV!UZ|9CD}s*AE=(*Z-rhxmA-aPz86II^@YpN7(Mw+#*tIqYSxy_B8p{(FzN? zuk#*&L>`^fz)PUdE0x*IOg=L`hBayx=V^-6kV{>6$y!&K3qR?j)ARZ;+PFtb2Ftm$|iW?~F EFV+^oQ2+n{ diff --git a/src/exchanges/deutsche_boerse.py b/src/exchanges/deutsche_boerse.py index db30015..73affb1 100644 --- a/src/exchanges/deutsche_boerse.py +++ b/src/exchanges/deutsche_boerse.py @@ -2,11 +2,17 @@ import requests import gzip import json import io +import time from datetime import datetime, timedelta, timezone from typing import List, Optional from .base import BaseExchange, Trade from bs4 import BeautifulSoup +# Rate-Limiting Konfiguration +RATE_LIMIT_DELAY = 0.5 # Sekunden zwischen Requests +RATE_LIMIT_RETRY_DELAY = 5 # Sekunden Wartezeit bei 429 +MAX_RETRIES = 3 # Maximale Wiederholungen bei 429 + # API URLs für Deutsche Börse API_URLS = { 'XETRA': 'https://mfs.deutsche-boerse.com/api/DETR-posttrade', @@ -94,49 +100,62 @@ class DeutscheBoerseBase(BaseExchange): def _download_and_parse_file(self, filename: str) -> List[Trade]: """Lädt eine JSON.gz Datei von der API herunter und parst die Trades""" trades = [] + full_url = f"{DOWNLOAD_BASE_URL}/{filename}" - try: - # Download-URL: https://mfs.deutsche-boerse.com/api/download/{filename} - full_url = f"{DOWNLOAD_BASE_URL}/{filename}" - - response = requests.get(full_url, headers=HEADERS, timeout=60) - - if response.status_code == 404: - # Datei nicht gefunden - normal für alte Dateien - return [] - - response.raise_for_status() - - # Gzip entpacken - with gzip.GzipFile(fileobj=io.BytesIO(response.content)) as f: - content = f.read().decode('utf-8') - - if not content.strip(): - # Leere Datei - return [] - - # NDJSON Format: Eine JSON-Zeile pro Trade - for line in content.strip().split('\n'): - if not line.strip(): + for retry in range(MAX_RETRIES): + try: + response = requests.get(full_url, headers=HEADERS, timeout=60) + + if response.status_code == 404: + # Datei nicht gefunden - normal für alte Dateien + return [] + + if response.status_code == 429: + # Rate-Limit erreicht - warten und erneut versuchen + wait_time = RATE_LIMIT_RETRY_DELAY * (retry + 1) + print(f"[{self.name}] Rate limited, waiting {wait_time}s...") + time.sleep(wait_time) continue - try: - record = json.loads(line) - trade = self._parse_trade_record(record) - if trade: - trades.append(trade) - except json.JSONDecodeError: + + response.raise_for_status() + + # Gzip entpacken + with gzip.GzipFile(fileobj=io.BytesIO(response.content)) as f: + content = f.read().decode('utf-8') + + if not content.strip(): + # Leere Datei + return [] + + # NDJSON Format: Eine JSON-Zeile pro Trade + for line in content.strip().split('\n'): + if not line.strip(): + continue + try: + record = json.loads(line) + trade = self._parse_trade_record(record) + if trade: + trades.append(trade) + except json.JSONDecodeError: + continue + except Exception: + continue + + # Erfolg - keine weitere Retry nötig + break + + except requests.exceptions.HTTPError as e: + if e.response.status_code == 429: + wait_time = RATE_LIMIT_RETRY_DELAY * (retry + 1) + print(f"[{self.name}] Rate limited, waiting {wait_time}s...") + time.sleep(wait_time) continue - except Exception as e: - continue - - if trades: - print(f"[{self.name}] Parsed {len(trades)} trades from {filename}") - - except requests.exceptions.HTTPError as e: - if e.response.status_code != 404: - print(f"[{self.name}] HTTP error downloading {filename}: {e}") - except Exception as e: - print(f"[{self.name}] Error downloading/parsing {filename}: {e}") + elif e.response.status_code != 404: + print(f"[{self.name}] HTTP error downloading {filename}: {e}") + break + except Exception as e: + print(f"[{self.name}] Error downloading/parsing {filename}: {e}") + break return trades @@ -274,13 +293,23 @@ class DeutscheBoerseBase(BaseExchange): print(f"[{self.name}] No files for target date found") return [] - # Alle passenden Dateien herunterladen und parsen + # Alle passenden Dateien herunterladen und parsen (mit Rate-Limiting) successful = 0 - for file in target_files: + total_files = len(target_files) + + for i, file in enumerate(target_files): trades = self._download_and_parse_file(file) if trades: all_trades.extend(trades) successful += 1 + + # Rate-Limiting: Pause zwischen Downloads + if i < total_files - 1: + time.sleep(RATE_LIMIT_DELAY) + + # Fortschritt alle 100 Dateien + if (i + 1) % 100 == 0: + print(f"[{self.name}] Progress: {i + 1}/{total_files} files, {len(all_trades)} trades so far") print(f"[{self.name}] Downloaded {successful} files, total {len(all_trades)} trades") return all_trades diff --git a/src/exchanges/gettex.py b/src/exchanges/gettex.py index c83c9b1..c0f352d 100644 --- a/src/exchanges/gettex.py +++ b/src/exchanges/gettex.py @@ -2,11 +2,15 @@ import requests import gzip import csv import io +import time from datetime import datetime, timedelta, timezone from typing import List, Optional from .base import BaseExchange, Trade from bs4 import BeautifulSoup +# Rate-Limiting +RATE_LIMIT_DELAY = 0.3 # Sekunden zwischen Requests + # Browser User-Agent für Zugriff (gettex prüft User-Agent!) HEADERS = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', @@ -411,11 +415,14 @@ class GettexExchange(BaseExchange): print(f"[{self.name}] Found {len(target_files)} files for target date from page") - # Lade Dateien von der Webseite - for f in target_files: + # Lade Dateien von der Webseite (mit Rate-Limiting) + for i, f in enumerate(target_files): trades = self._download_file_by_url(f['url'], f['filename']) if trades: all_trades.extend(trades) + # Rate-Limiting + if i < len(target_files) - 1: + time.sleep(RATE_LIMIT_DELAY) # Fallback: Versuche erwartete Dateinamen if not all_trades: