From 445eb0b482808667bf75ebf65a76723991feb2eb Mon Sep 17 00:00:00 2001 From: Melchior Reimers Date: Sun, 25 Jan 2026 15:00:53 +0100 Subject: [PATCH] fixing dashboard --- requirements.txt | 1 + .../__pycache__/fetcher.cpython-313.pyc | Bin 0 -> 8944 bytes src/metadata/fetcher.py | 51 ++++++++++++++---- test_sector_fetch.py | 35 ++++++++++++ 4 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 src/metadata/__pycache__/fetcher.cpython-313.pyc create mode 100644 test_sector_fetch.py diff --git a/requirements.txt b/requirements.txt index 6f16e08..5fbc4bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ fastapi uvicorn pandas python-multipart +yfinance diff --git a/src/metadata/__pycache__/fetcher.cpython-313.pyc b/src/metadata/__pycache__/fetcher.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f0d38892d21590642ba95454ce5b29c31929617 GIT binary patch literal 8944 zcmeHMTWlLwdOmaGZAgj|b(d_7?xrn?k|kLeTed05xR_08aalpyr4_#u;+bm2vmKLEGYYX5d?4dUre zm>nm{bd{Pc(95|w^kH~IC#ly-mhbFk{ZK=^dLSKp{s26iH0Z3RknkiAb3Ku ze&=Xr-jW^7#4`s{+Z%+o7e9&x5>(cR*gu=R;n?7eHQm zwTyT6UPj1gg1Jj;P-l8&9GWF&30L74_<20qQGdiI`y;_Q!6kFeT>TNzmo0{bn?fk( zc}|dJ-z;==kt(A#s)m`cH0O(`_6tJ97w|=VgFay{EUL!A{%J3NZc=4@fw`clvhaM- z!}D)WOb)8dA1?@UMC&s$!Cz91+H>HVKfOfl#{fA6|1YlTf%RcP>814gi{ynOgjmGE8zNN)-|9wn zP}ZMkP&Jy$3c3;bY7r9YsjP5M=I>bwJo08QZAa~7M$|b}1)?L;9vY9Vmu83=)JIjJ z8Ny{5k8-jQ68sS^5R@Z95eh*$C~`AWc#exmzJMUVG+vKH=6l`lmzLh1*49?Ha9i+e z$0;rTf>aIjKFK#HtF&(+a{Y@9Sc7WBG9e2Q2_IMV&^bv8OWceQ@m~*$v)qChgrnz1 z`O$H?mvfm_lO$-eAgk0Yyo6O=6BCd3(um5+5np6Mp7w_Wf=d693}R+JciS(_M}kl^ zgu=74f}}D6p2NjRdH7`*P@WHqGQ5CSqEb@$mP!aRJ_-&e^@`go3zF=f6GE`4up|kf z3-H`BD}dkz-99lAoRxexgONqI?3aA7bT{4&SpRfD6Lz;O`Qde4{@vQg2~x}aqUwNe znAX2>8gHd6)xflPUGis;EuoaPW$D}wX<&M{ofY>_y?bi4@zch5+1gQs>w9oZx#(4f z-q`$MAmJ32O)1v7#a1QQs@3)++mM3(efRp_JF`5PYB>7I^^dN{7uRbx8~T#Yk;e^v zG5Nvkzb<}nXk{Qq{B$VM(3fzIDAti323Z?-EXZnK8vXrqBPtvvzTRdFUmyZjV6xw} z-mxkr9c!TP0W;4!Dkhta) z1NkGu(%(WoVoUp7)FT%IWk0-=OhGn}I%eIjz_`j5GYg)F=yRuC(1<&&G z-!bnnPk+(oFiegW)RaAqOIhZpAkAh_9Ngc8Pq7jJ3ed>F_K zPqq>w!Bbu!87-un5Xl2Ei@99Q8+c<6EmB@;67Te$#G5o+8pCKYiRavtvbaQh=?uR3 zIuM#NXF7Vbgoiix{uD?-J>Y}IOY>%I2v5SluM$R2Kl2t`3%CVBBr-hfWpED-u>cjl zP=nhsHq~i}HERcS%F;7HCm3Bc?BdlVHhFszZ}lQTXq(pn7?oX}hEaCW;4#eFdF?j` zq4J{5V5+%^c|jZ+9UkR87e~2Bn49y>YY2kNk)Z#EAaU~I+_i8hY8?oR5kZVJUs{|O zq7L8ud?@JmVYF~-=wTckoW|G#bjA=2;7+PB70fe%2}<{b}9uYmW`>y zjcbDmXOB|QyKKaI-S=*vQazNaZ+akoNfCu(#IotxcLCMYJm-g0Z1891RnO0C@rJd* z_59Cc|Iw^0>75O9mZk!l32&`YXV% zB8*nuS+qJD?|`AmwYIfdrRc<#sY@|+rPw?TvASQ%*eLzj)8aXe9(EXTev-ubX-j`I z^~hAwe}sB;gn{z@YW*1AejljVK`QJ;8Iq+@%Y9TBQ^u@nf)eqBm(B7y1nmLh>c@ldlgul?pI5)E3C@TD@ar}mJtRmMD4;+XL4Vxd zg??%AG*AzT3Qsllu$qDL{u1?Cb29HBk8&oBjRdDg!F}0S<53b#z3=%VZ_{ zN4xN_?|atPtb-&PyoP=I*mnEu+ipLgJ&z|dPMDYA^BQ4O{+Ba34wxzbm6`JKOkgXr zu2_y$FnLYcJvtA-oIR+0-N-_(381k^JLw!~oWgWYOI}mf-k81S?8v~>F5lye@j4Ee zp~P#38A|ug0Q;4XEfG)l-U3RngujO`&(vZW6)gwK-Os<5++93K?p~2AFc4E%^R(n; zrZ9!oQs$(ydjJMMLs0gWo{AjFWxcx*8WX(7>cj@Vw^EZDq7DK9b!d<4-2mjqn6tn8 zk(&a%v%J%5{FY6IcRj{oV!Q|(a479sX)0;iBP!V0Y5RL{zA)mXmrIkB!!RHT(k+6LIBsU6#FT_CwH^Fn4xwh8n zw$AA;7v}-)C%h?K8w*COA2R0LU5)fEX`ty=)T$JhUaEU@Q0;t z8K{anhR>fH9pd)jZ`3vfI~d4t;ix6M=bXNg>Gm|-M;Wa>TF&e6pAF9J4jZk{d^gM; z(a$F+$Z0m*(&qMs=CAwQswI13vUCNe9E2=SF zjge|#uUfPiN(56Dl@39S3P#Zc$CQySoerC|lbK$SLJ;l|WSxD}c#AcTp)n6_CsZ~q zYq+m;4wfO~7#n8FnWQsjyUv)i(_DaF!?^n*9N!Y!rY(Oy_NQY?K}$RmpHUp0oAili zj>43qEaj|B6&+3$RD8)A9EPQlokCW=sri#yRT-q2?3i)O8 zw!Qdn=$%lkXI)Cz`5O{T} zq$1|IckBM!_ugJTwN{WU=~$stwT+*2f7BfxS-Y03?b)jBOVswQ-`F^wtR3H~olMkD zD*P2?Dv+!dR!p(tgoDcjl5VdTBpXg`HJnW}oPBUud7V!-T-s{5oM^bLT)CzM=8_E} zyit~L)b3c!%$e={WAVXce%rF)DVx7-Ex0>&XDrqnH>}={N7w7ti>s&8fHz<{7HXUby?wcGd=7h047EBr& zcSwu*B9W@}JUFt^lj!#fcfwvAe)Y<$Z)p;#xtw_2L`X2*!Tzm#7C5&U?-G0dS~b$Jd$ zl=;_Hp1x5A{f4uQIH=zotr%gc4VHoOhSM@iQX7XVM*66YJ_gF4GnUbA>hrwAqwUn^ z?F^J(A+X@0=#h=T(IOi_NHrH_GB^4GU2wveEg2Zzw;jl9rXAcPps<)?fJ20MqgKMP z5-m0kR28DuS{(z?2?ijP^%9U5E$tUKf%^c*t~&*9 z<+oE_17Cn$k6ovVcLGT^dX4*CF!6;a;fry!{eWmj5xfu4lD+9wT0e_s5x{)*x(OV- z8rTWb0b>_qI+*JoXmNa#$25yU7k+n1gsWs+M+<_Bl2SECX&TFf!Z>Q@1`BnUaEgavR; z12+7XP{y3~i+_SFT5?Vc{>utzp`m5Wz4-d27PP=p&wAf$xm6jods+@Tr$$^gPIt6iUV#a(MBmFhDO0vnee-q^f6 zop8RDdtnewvTa%8a(u1(mwli0t+|!5{@+>$b|_>${`K$vz$HM!)T7xdaDL0+vme?# zCH)QPQG;cmfO^zgI$)!IWn-Z1hsJEeE-7|(U+DHDhc55y0B7784m(iT0B9-Wf{JL$ z_^mpyByh(@X7AVlxx+PH0_&?5sf5^^mxNn22#MocHIV{~LuK6rxEMWPO=M2=l*+6R zYzW*bW}Y~R$3m181QPDzu*KBUZmd;grf^hVuVr@M%DK1O$7bYSPjl!3FdV%8iJ}A6 z`#%}qu)CHEe-GkJa_}#bGZ0rCkCt9YUuALnT`mXLZc;=Dv_uU$v5XdTqHs&UwZpNa z^hH3+7qImx>KqT}G|ylv+7fiYDY(ozoDb|ClE7C(QY+3dq0+Pq^cSM15Elvw!aM|$ zthv``YTlkZ2bBn_W>`!51_F~o&G^Vz{B%{;2^>A=gJ9`fPMzjm%( z`K!YU*Q&G*DW$`k){!Tr)hqONN&RXdS#k_Yjzh7)rlWS-QF!0-t|iv?7q;g{R95qq z1=*aKDk-JSo7R@!nXGp$?^sqkHch2pRv|X;_s>nJ=ruwHF-CuD9&phAXn;&LO;6)b znVxo)>(=lG(5@Qs-6vR8l?=VJ~`+I>k6@Jl^z`*no!LU=sT}nlxl~e z8sJ$FZ%TN>B=CXLcS3kp?XprkBbB-)cS!YkCQ~;4)`h#^aIr?tcTZ5;WxP%dC)3lv zKrf&V(vQJ(YAzgD2nnYp5$dqG$o~Kt@Fs$Ig7Tgq_6f2-L1rjFL&h!Se2kp2#=mU( zph{im!xMWP(9e0E81Y@?cn ziG5mk?C;p+u9dNbg?ms4?b-f|NsDXA_?$Kq_UFYG!v3G-h&cQN6+NdNgyVUg8M5LN nMB(#!!a?*tugfROb{Y_EU5+NHvVW#ce>Mb$lP;I$VEO+IR~uaH literal 0 HcmV?d00001 diff --git a/src/metadata/fetcher.py b/src/metadata/fetcher.py index 0258602..0b0eda0 100644 --- a/src/metadata/fetcher.py +++ b/src/metadata/fetcher.py @@ -35,7 +35,38 @@ def get_processed_isins(): return [] return [] + from bs4 import BeautifulSoup +import yfinance as yf + +def fetch_ticker_from_openfigi(isin): + """Use OpenFIGI API to map ISIN to ticker symbol""" + try: + headers = {'Content-Type': 'application/json'} + payload = [{'idType': 'ID_ISIN', 'idValue': isin}] + response = requests.post('https://api.openfigi.com/v3/mapping', + json=payload, headers=headers, timeout=10) + if response.status_code == 200: + data = response.json() + if data and len(data) > 0 and 'data' in data[0]: + # Get the first result's ticker + for item in data[0]['data']: + if 'ticker' in item: + return item['ticker'] + except Exception as e: + logger.error(f"OpenFIGI error for {isin}: {e}") + return None + +def fetch_sector_from_yfinance(ticker): + """Use yfinance to get sector information from ticker symbol""" + try: + stock = yf.Ticker(ticker) + info = stock.info + if info and 'sector' in info: + return info['sector'] + except Exception as e: + logger.error(f"yfinance error for {ticker}: {e}") + return None def fetch_metadata(isin): logger.info(f"Fetching metadata for ISIN: {isin}") @@ -65,19 +96,17 @@ def fetch_metadata(isin): except Exception as e: logger.error(f"GLEIF error for {isin}: {e}") - # 2. Yahoo Finance for Sector + # 2. Sector from OpenFIGI + yfinance try: - # We use the lookup URL as discussed - yahoo_url = f"https://finance.yahoo.com/lookup/?s={isin}" - res = requests.get(yahoo_url, headers=headers, timeout=10) - if res.status_code == 200: - soup = BeautifulSoup(res.text, 'html.parser') - # Look for the sector link in the results table - sector_link = soup.find('a', href=lambda x: x and '/sector/' in x) - if sector_link: - metadata['sector'] = sector_link.text.strip() + ticker = fetch_ticker_from_openfigi(isin) + if ticker: + logger.info(f"Found ticker {ticker} for ISIN {isin}") + sector = fetch_sector_from_yfinance(ticker) + if sector: + metadata['sector'] = sector + logger.info(f"Found sector {sector} for {isin}") except Exception as e: - logger.error(f"Yahoo sector error for {isin}: {e}") + logger.error(f"Sector fetching error for {isin}: {e}") # 3. Continent mapping from Country Code if metadata['country'] != 'Unknown': diff --git a/test_sector_fetch.py b/test_sector_fetch.py new file mode 100644 index 0000000..0f59ae9 --- /dev/null +++ b/test_sector_fetch.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Test script to verify the improved sector metadata fetching""" + +import sys +sys.path.insert(0, '/Users/melchiorreimers/.gemini/antigravity/scratch/trading_daemon/src') + +from metadata.fetcher import fetch_ticker_from_openfigi, fetch_sector_from_yfinance, fetch_metadata + +# Test ISINs from different regions and sectors +test_isins = [ + 'US69553P1003', # PagerDuty (US, Technology) + 'DE000LS1LUS9', # German security + 'US0378331005', # Apple (US, Technology) +] + +print("Testing improved sector metadata fetching...\n") + +for isin in test_isins: + print(f"Testing ISIN: {isin}") + + # Test ticker lookup + ticker = fetch_ticker_from_openfigi(isin) + print(f" Ticker: {ticker}") + + # Test sector lookup if ticker found + if ticker: + sector = fetch_sector_from_yfinance(ticker) + print(f" Sector: {sector}") + + print() + +print("\nFull metadata test for US69553P1003:") +metadata = fetch_metadata('US69553P1003') +for key, value in metadata.items(): + print(f" {key}: {value}")