fixing dashboard

src/metadata/fetcher.py

@@ -35,7 +35,38 @@ def get_processed_isins():
         return []
     return []
 
+
 from bs4 import BeautifulSoup
+import yfinance as yf
+
+def fetch_ticker_from_openfigi(isin):
+    """Use OpenFIGI API to map ISIN to ticker symbol"""
+    try:
+        headers = {'Content-Type': 'application/json'}
+        payload = [{'idType': 'ID_ISIN', 'idValue': isin}]
+        response = requests.post('https://api.openfigi.com/v3/mapping', 
+                                json=payload, headers=headers, timeout=10)
+        if response.status_code == 200:
+            data = response.json()
+            if data and len(data) > 0 and 'data' in data[0]:
+                # Get the first result's ticker
+                for item in data[0]['data']:
+                    if 'ticker' in item:
+                        return item['ticker']
+    except Exception as e:
+        logger.error(f"OpenFIGI error for {isin}: {e}")
+    return None
+
+def fetch_sector_from_yfinance(ticker):
+    """Use yfinance to get sector information from ticker symbol"""
+    try:
+        stock = yf.Ticker(ticker)
+        info = stock.info
+        if info and 'sector' in info:
+            return info['sector']
+    except Exception as e:
+        logger.error(f"yfinance error for {ticker}: {e}")
+    return None
 
 def fetch_metadata(isin):
     logger.info(f"Fetching metadata for ISIN: {isin}")
@@ -65,19 +96,17 @@ def fetch_metadata(isin):
     except Exception as e:
         logger.error(f"GLEIF error for {isin}: {e}")
 
-    # 2. Yahoo Finance for Sector
+    # 2. Sector from OpenFIGI + yfinance
     try:
-        # We use the lookup URL as discussed
-        yahoo_url = f"https://finance.yahoo.com/lookup/?s={isin}"
-        res = requests.get(yahoo_url, headers=headers, timeout=10)
-        if res.status_code == 200:
-            soup = BeautifulSoup(res.text, 'html.parser')
-            # Look for the sector link in the results table
-            sector_link = soup.find('a', href=lambda x: x and '/sector/' in x)
-            if sector_link:
-                metadata['sector'] = sector_link.text.strip()
+        ticker = fetch_ticker_from_openfigi(isin)
+        if ticker:
+            logger.info(f"Found ticker {ticker} for ISIN {isin}")
+            sector = fetch_sector_from_yfinance(ticker)
+            if sector:
+                metadata['sector'] = sector
+                logger.info(f"Found sector {sector} for {isin}")
     except Exception as e:
-        logger.error(f"Yahoo sector error for {isin}: {e}")
+        logger.error(f"Sector fetching error for {isin}: {e}")
 
     # 3. Continent mapping from Country Code
     if metadata['country'] != 'Unknown':