diff --git a/README.md b/README.md index 671e8cd..2811c61 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,63 @@ Use `poetry run pyth-observer --help` for documentation on arguments and environ To run tests, use `poetry run pytest`. +## Building CoinGecko Mapping + +The `scripts/build_coingecko_mapping.py` script automatically generates a CoinGecko mapping file by fetching all price feeds from the Pyth Hermes API and matching them with CoinGecko's coin list using fuzzy matching. + +### Basic Usage + +```sh +# Generate a new mapping file +poetry run python scripts/build_coingecko_mapping.py + +# Compare with existing mapping file +poetry run python scripts/build_coingecko_mapping.py -e sample.coingecko.yaml + +# Specify custom output file +poetry run python scripts/build_coingecko_mapping.py -o my_mapping.json + +# Skip price validation (faster, but less thorough) +poetry run python scripts/build_coingecko_mapping.py --no-validate-prices + +# Adjust maximum price deviation threshold (default: 10.0%) +poetry run python scripts/build_coingecko_mapping.py --max-price-deviation 5.0 +``` + +### How It Works + +1. **Fetches Pyth Price Feeds**: Retrieves all price feeds from `https://hermes.pyth.network/v2/price_feeds` +2. **Extracts Crypto Symbols**: Filters for Crypto asset types and extracts symbols (e.g., "Crypto.BTC/USD") +3. **Matches with CoinGecko**: Uses multiple matching strategies: + - Exact symbol match (case-insensitive) + - Fuzzy symbol matching + - Fuzzy name matching based on Pyth description +4. **Validates Mappings**: Compares generated mappings against known correct mappings +5. **Validates Prices** (optional): Compares prices from Hermes and CoinGecko to detect mismatches +6. **Generates Warnings**: Flags symbols that need manual review: + - Low-confidence fuzzy matches (shows similarity score) + - Symbols with no matches found + - Price deviations between sources + +### Output + +The script generates a JSON file in the format: +```json +{ + "Crypto.BTC/USD": "bitcoin", + "Crypto.ETH/USD": "ethereum", + ... +} +``` + +The script provides a summary showing: +- Total symbols mapped +- Exact matches (100% confidence) +- Fuzzy matches (needs review) +- No matches found + +Review the warnings output to manually verify and adjust any low-confidence matches before using the generated mapping file. + ## Configuration See `sample.config.yaml` for configuration options. diff --git a/pyproject.toml b/pyproject.toml index c6e714f..d5237b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ ignore_missing_imports = true [tool.poetry] name = "pyth-observer" -version = "2.1.4" +version = "3.0.0" description = "Alerts and stuff" authors = [] readme = "README.md" @@ -50,4 +50,4 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.poetry.requires-plugins] -poetry-plugin-export = ">=1.8" \ No newline at end of file +poetry-plugin-export = ">=1.8" diff --git a/pyth_observer/__init__.py b/pyth_observer/__init__.py index ab3e286..1077fd3 100644 --- a/pyth_observer/__init__.py +++ b/pyth_observer/__init__.py @@ -2,10 +2,9 @@ import os from typing import Any, Dict, List, Literal, Tuple -from base58 import b58decode from loguru import logger from pythclient.market_schedule import MarketSchedule -from pythclient.pythaccounts import PythPriceAccount, PythPriceType, PythProductAccount +from pythclient.pythaccounts import PythProductAccount from pythclient.pythclient import PythClient from pythclient.solana import ( SOLANA_DEVNET_HTTP_ENDPOINT, @@ -21,9 +20,7 @@ from pyth_observer.check import State from pyth_observer.check.price_feed import PriceFeedState from pyth_observer.check.publisher import PublisherState -from pyth_observer.coingecko import Symbol, get_coingecko_prices -from pyth_observer.crosschain import CrosschainPrice -from pyth_observer.crosschain import CrosschainPriceObserver as Crosschain +from pyth_observer.coingecko import get_coingecko_prices from pyth_observer.dispatch import Dispatch from pyth_observer.metrics import metrics from pyth_observer.models import Publisher @@ -57,7 +54,7 @@ def __init__( self, config: Dict[str, Any], publishers: Dict[str, Publisher], - coingecko_mapping: Dict[str, Symbol], + coingecko_mapping: Dict[str, str], ) -> None: self.config = config self.dispatch = Dispatch(config, publishers) @@ -71,8 +68,6 @@ def __init__( rate_limit=int(config["network"]["request_rate_limit"]), period=float(config["network"]["request_rate_period"]), ) - self.crosschain = Crosschain(self.config["network"]["crosschain_endpoint"]) - self.crosschain_throttler = Throttler(rate_limit=1, period=1) self.coingecko_mapping = coingecko_mapping metrics.set_observer_info( @@ -89,7 +84,9 @@ async def run(self) -> None: products = await self.get_pyth_products() coingecko_prices, coingecko_updates = await self.get_coingecko_prices() - crosschain_prices = await self.get_crosschain_prices() + await self.refresh_all_pyth_prices() + + logger.info("Refreshed all state: products, coingecko, pyth") health_server.observer_ready = True @@ -98,7 +95,7 @@ async def run(self) -> None: for product in products: # Skip tombstone accounts with blank metadata - if "base" not in product.attrs: + if "symbol" not in product.attrs: continue if not product.first_price_account_key: @@ -108,11 +105,7 @@ async def run(self) -> None: # for each price account) and a list of publisher states (one # for each publisher). states: List[State] = [] - price_accounts = await self.get_pyth_prices(product) - - crosschain_price = crosschain_prices.get( - b58decode(product.first_price_account_key.key).hex(), None - ) + price_accounts = product.prices for _, price_account in price_accounts.items(): # Handle potential None for min_publishers @@ -146,11 +139,12 @@ async def run(self) -> None: latest_trading_slot=price_account.last_slot, price_aggregate=price_account.aggregate_price_info.price, confidence_interval_aggregate=price_account.aggregate_price_info.confidence_interval, - coingecko_price=coingecko_prices.get(product.attrs["base"]), + coingecko_price=coingecko_prices.get( + product.attrs["symbol"] + ), coingecko_update=coingecko_updates.get( - product.attrs["base"] + product.attrs["symbol"] ), - crosschain_price=crosschain_price, ) states.append(price_feed_state) @@ -231,21 +225,19 @@ async def get_pyth_products(self) -> List[PythProductAccount]: ).inc() raise - async def get_pyth_prices( - self, product: PythProductAccount - ) -> Dict[PythPriceType, PythPriceAccount]: - logger.debug("Fetching Pyth price accounts...") + async def refresh_all_pyth_prices(self) -> None: + """Refresh all Pyth prices once for all products.""" + logger.debug("Refreshing all Pyth price accounts...") try: async with self.pyth_throttler: with metrics.time_operation( metrics.api_request_duration, service="pyth", endpoint="prices" ): - result = await product.refresh_prices() + await self.pyth_client.refresh_all_prices() metrics.api_request_total.labels( service="pyth", endpoint="prices", status="success" ).inc() - return result except Exception: metrics.api_request_total.labels( service="pyth", endpoint="prices", status="error" @@ -285,19 +277,3 @@ async def get_coingecko_prices( updates[symbol] = data[symbol]["last_updated_at"] return (prices, updates) - - async def get_crosschain_prices(self) -> Dict[str, CrosschainPrice]: - try: - with metrics.time_operation( - metrics.api_request_duration, service="crosschain", endpoint="prices" - ): - result = await self.crosschain.get_crosschain_prices() - metrics.api_request_total.labels( - service="crosschain", endpoint="prices", status="success" - ).inc() - return result - except Exception: - metrics.api_request_total.labels( - service="crosschain", endpoint="prices", status="error" - ).inc() - raise diff --git a/pyth_observer/alert_utils.py b/pyth_observer/alert_utils.py new file mode 100644 index 0000000..c42cfc9 --- /dev/null +++ b/pyth_observer/alert_utils.py @@ -0,0 +1,18 @@ +""" +Utility functions for alert identification and management. +""" + +from pyth_observer.check import Check +from pyth_observer.check.publisher import PublisherState + + +def generate_alert_identifier(check: Check) -> str: + """ + Generate a unique alert identifier for a check. + This is a shared function to ensure consistency across the codebase. + """ + alert_identifier = f"{check.__class__.__name__}-{check.state().symbol}" + state = check.state() + if isinstance(state, PublisherState): + alert_identifier += f"-{state.publisher_name}" + return alert_identifier diff --git a/pyth_observer/check/price_feed.py b/pyth_observer/check/price_feed.py index e0c2160..d9b431b 100644 --- a/pyth_observer/check/price_feed.py +++ b/pyth_observer/check/price_feed.py @@ -4,13 +4,10 @@ from typing import Any, Dict, Optional, Protocol, runtime_checkable from zoneinfo import ZoneInfo -import arrow from pythclient.market_schedule import MarketSchedule from pythclient.pythaccounts import PythPriceStatus from pythclient.solana import SolanaPublicKey -from pyth_observer.crosschain import CrosschainPrice - @dataclass class PriceFeedState: @@ -25,7 +22,6 @@ class PriceFeedState: confidence_interval_aggregate: float coingecko_price: Optional[float] coingecko_update: Optional[int] - crosschain_price: Optional[CrosschainPrice] PriceFeedCheckConfig = Dict[str, str | float | int | bool] @@ -93,7 +89,7 @@ def error_message(self) -> Dict[str, Any]: class PriceFeedCoinGeckoCheck(PriceFeedCheck): def __init__(self, state: PriceFeedState, config: PriceFeedCheckConfig) -> None: self.__state = state - self.__max_deviation: int = int(config["max_deviation"]) # Percentage + self.__max_deviation: float = float(config["max_deviation"]) # Percentage self.__max_staleness: int = int(config["max_staleness"]) # Seconds def state(self) -> PriceFeedState: @@ -112,6 +108,10 @@ def run(self) -> bool: if self.__state.status != PythPriceStatus.TRADING: return True + # Skip if CoinGecko price is zero + if self.__state.coingecko_price == 0: + return True + deviation = ( abs(self.__state.price_aggregate - self.__state.coingecko_price) / self.__state.coingecko_price @@ -163,124 +163,8 @@ def error_message(self) -> Dict[str, Any]: } -class PriceFeedCrossChainOnlineCheck(PriceFeedCheck): - def __init__(self, state: PriceFeedState, config: PriceFeedCheckConfig) -> None: - self.__state = state - self.__max_staleness: int = int(config["max_staleness"]) - - def state(self) -> PriceFeedState: - return self.__state - - def run(self) -> bool: - # Skip if not trading - if self.__state.status != PythPriceStatus.TRADING: - return True - - market_open = self.__state.schedule.is_market_open( - datetime.now(ZoneInfo("America/New_York")), - ) - - # Skip if not trading hours (for equities) - if not market_open: - return True - - # Price should exist, it fails otherwise - if not self.__state.crosschain_price: - return False - - # Skip if publish time is zero - if not self.__state.crosschain_price["publish_time"]: - return True - - staleness = ( - self.__state.crosschain_price["snapshot_time"] - - self.__state.crosschain_price["publish_time"] - ) - - # Pass if current staleness is less than `max_staleness` - if staleness < self.__max_staleness: - return True - - # Fail - return False - - def error_message(self) -> Dict[str, Any]: - if self.__state.crosschain_price: - publish_time = arrow.get(self.__state.crosschain_price["publish_time"]) - else: - publish_time = arrow.get(0) - - return { - "msg": f"{self.__state.symbol} isn't online at the price service.", - "type": "PriceFeedCrossChainOnlineCheck", - "symbol": self.__state.symbol, - "last_publish_time": publish_time.format("YYYY-MM-DD HH:mm:ss ZZ"), - } - - -class PriceFeedCrossChainDeviationCheck(PriceFeedCheck): - def __init__(self, state: PriceFeedState, config: PriceFeedCheckConfig) -> None: - self.__state = state - self.__max_deviation: int = int(config["max_deviation"]) - self.__max_staleness: int = int(config["max_staleness"]) - - def state(self) -> PriceFeedState: - return self.__state - - def run(self) -> bool: - # Skip if does not exist - if not self.__state.crosschain_price: - return True - - # Skip if not trading - if self.__state.status != PythPriceStatus.TRADING: - return True - - market_open = self.__state.schedule.is_market_open( - datetime.now(ZoneInfo("America/New_York")), - ) - - # Skip if not trading hours (for equities) - if not market_open: - return True - - staleness = int(time.time()) - self.__state.crosschain_price["publish_time"] - - # Skip if price is stale - if staleness > self.__max_staleness: - return True - - deviation = ( - abs(self.__state.crosschain_price["price"] - self.__state.price_aggregate) - / self.__state.price_aggregate - ) * 100 - - # Pass if price isn't higher than maxium deviation - if deviation < self.__max_deviation: - return True - - # Fail - return False - - def error_message(self) -> Dict[str, Any]: - # It can never happen because of the check logic but linter could not understand it. - price = ( - self.__state.crosschain_price["price"] - if self.__state.crosschain_price - else None - ) - return { - "msg": f"{self.__state.symbol} is too far at the price service.", - "type": "PriceFeedCrossChainDeviationCheck", - "symbol": self.__state.symbol, - "price": self.__state.price_aggregate, - "price_at_price_service": price, - } - - PRICE_FEED_CHECKS = [ PriceFeedCoinGeckoCheck, - PriceFeedCrossChainDeviationCheck, - PriceFeedCrossChainOnlineCheck, + PriceFeedConfidenceIntervalCheck, PriceFeedOfflineCheck, ] diff --git a/pyth_observer/check/publisher.py b/pyth_observer/check/publisher.py index 90efc9d..161f5cb 100644 --- a/pyth_observer/check/publisher.py +++ b/pyth_observer/check/publisher.py @@ -70,7 +70,7 @@ def error_message(self) -> Dict[str, Any]: class PublisherWithinAggregateConfidenceCheck(PublisherCheck): def __init__(self, state: PublisherState, config: PublisherCheckConfig) -> None: self.__state = state - self.__max_interval_distance: int = int(config["max_interval_distance"]) + self.__max_interval_distance: float = float(config["max_interval_distance"]) def state(self) -> PublisherState: return self.__state @@ -94,6 +94,11 @@ def run(self) -> bool: return True diff = self.__state.price - self.__state.price_aggregate + + # Skip if confidence interval aggregate is zero + if self.__state.confidence_interval_aggregate == 0: + return True + intervals_away = abs(diff / self.__state.confidence_interval_aggregate) # Pass if price diff is less than max interval distance @@ -105,7 +110,11 @@ def run(self) -> bool: def error_message(self) -> Dict[str, Any]: diff = self.__state.price - self.__state.price_aggregate - intervals_away = abs(diff / self.__state.confidence_interval_aggregate) + if self.__state.confidence_interval_aggregate == 0: + intervals_away = abs(diff) + else: + intervals_away = abs(diff / self.__state.confidence_interval_aggregate) + return { "msg": f"{self.__state.publisher_name} price is {intervals_away} times away from confidence.", "type": "PublisherWithinAggregateConfidenceCheck", @@ -197,7 +206,9 @@ def error_message(self) -> Dict[str, Any]: class PublisherPriceCheck(PublisherCheck): def __init__(self, state: PublisherState, config: PublisherCheckConfig) -> None: self.__state = state - self.__max_aggregate_distance: int = int(config["max_aggregate_distance"]) # % + self.__max_aggregate_distance: float = float( + config["max_aggregate_distance"] + ) # % self.__max_slot_distance: int = int(config["max_slot_distance"]) # Slots def state(self) -> PublisherState: @@ -218,7 +229,7 @@ def run(self) -> bool: return True # Skip if published price is zero - if self.__state.price == 0: + if self.__state.price == 0 or self.__state.price_aggregate == 0: return True deviation = (self.ci_adjusted_price_diff() / self.__state.price_aggregate) * 100 @@ -231,7 +242,13 @@ def run(self) -> bool: return False def error_message(self) -> Dict[str, Any]: - deviation = (self.ci_adjusted_price_diff() / self.__state.price_aggregate) * 100 + if self.__state.price_aggregate == 0: + deviation = self.ci_adjusted_price_diff() + else: + deviation = ( + self.ci_adjusted_price_diff() / self.__state.price_aggregate + ) * 100 + return { "msg": f"{self.__state.publisher_name} price is too far from aggregate price.", "type": "PublisherPriceCheck", diff --git a/pyth_observer/coingecko.py b/pyth_observer/coingecko.py index 9841c35..b73f2d8 100644 --- a/pyth_observer/coingecko.py +++ b/pyth_observer/coingecko.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, TypedDict +from typing import Any, Dict from loguru import logger from pycoingecko import CoinGeckoAPI @@ -6,20 +6,15 @@ from throttler import throttle -class Symbol(TypedDict): - api: str - market: str - - # CoinGecko free API limit: 10-50 (varies) https://www.coingecko.com/en/api/pricing # However prices are updated every 1-10 minutes: https://www.coingecko.com/en/faq # Hence we only have to query once every minute. -@throttle(rate_limit=1, period=60) +@throttle(rate_limit=1, period=10) async def get_coingecko_prices( - mapping: Dict[str, Symbol], + symbol_to_ticker: Dict[str, str], ) -> Dict[str, Dict[str, Any]]: - inverted_mapping = {mapping[x]["api"]: x for x in mapping} - ids = [mapping[x]["api"] for x in mapping] + ticker_to_symbol = {v: k for k, v in symbol_to_ticker.items()} + ids = list(ticker_to_symbol.keys()) try: prices = CoinGeckoAPI().get_price( @@ -32,6 +27,4 @@ async def get_coingecko_prices( ) prices = {} - # remap to symbol -> prices - prices_mapping = {inverted_mapping[x]: prices[x] for x in prices} - return prices_mapping + return {ticker_to_symbol[x]: prices[x] for x in prices} diff --git a/pyth_observer/crosschain.py b/pyth_observer/crosschain.py deleted file mode 100644 index 2477d1c..0000000 --- a/pyth_observer/crosschain.py +++ /dev/null @@ -1,63 +0,0 @@ -import time -from typing import Dict, TypedDict - -import requests -from aiohttp import ClientSession -from loguru import logger -from more_itertools import chunked -from throttler import throttle - - -class CrosschainPrice(TypedDict): - price: float - conf: float - publish_time: int # UNIX timestamp - snapshot_time: int # UNIX timestamp - - -class CrosschainPriceObserver: - def __init__(self, url: str) -> None: - self.url = url - self.valid = self.is_endpoint_valid() - - def is_endpoint_valid(self) -> bool: - try: - return requests.head(self.url).status_code == 200 - except requests.ConnectionError: - logger.error("failed to connect to cross-chain api") - return False - - @throttle(rate_limit=1, period=1) - async def get_crosschain_prices(self) -> Dict[str, CrosschainPrice]: - async with ClientSession( - headers={"content-type": "application/json"} - ) as session: - price_feeds_index_url = f"{self.url}/v2/price_feeds" - - async with session.get(price_feeds_index_url) as response: - price_feeds_index = await response.json() - - price_feeds = [] - - for feeds in chunked(price_feeds_index, 25): - price_feeds_url = f"{self.url}/v2/updates/price/latest" - - # aiohttp does not support encoding array params using PHP-style `ids=[]` - # naming, so we encode it manually and append to the URL. - query_string = "?" + "&".join(f"ids[]={v['id']}" for v in feeds) - async with session.get( - price_feeds_url + query_string, - ) as response: - response_json = await response.json() - price_feeds.extend(response_json["parsed"]) - - # Return a dictionary of id -> {price, conf, expo} for fast lookup - return { - data["id"]: { - "price": int(data["price"]["price"]) * 10 ** data["price"]["expo"], - "conf": int(data["price"]["conf"]) * 10 ** data["price"]["expo"], - "publish_time": data["price"]["publish_time"], - "snapshot_time": int(time.time()), - } - for data in price_feeds - } diff --git a/pyth_observer/dispatch.py b/pyth_observer/dispatch.py index 5bc3be0..698e5b4 100644 --- a/pyth_observer/dispatch.py +++ b/pyth_observer/dispatch.py @@ -3,10 +3,11 @@ import os from copy import deepcopy from datetime import datetime, timedelta -from typing import Any, Awaitable, Dict, List +from typing import Any, Awaitable, Dict, List, Optional, TypedDict from loguru import logger +from pyth_observer.alert_utils import generate_alert_identifier from pyth_observer.check import Check, State from pyth_observer.check.price_feed import PRICE_FEED_CHECKS, PriceFeedState from pyth_observer.check.publisher import PUBLISHER_CHECKS, PublisherState @@ -24,6 +25,27 @@ assert ZendutyEvent +class AlertInfo(TypedDict): + """ + Information about an open alert tracked for threshold-based alerting. + + Fields: + type: The check class name (e.g., "PublisherOfflineCheck") + window_start: ISO format datetime string marking the start of the current 5-minute window + failures: Number of failures in the current 5-minute window + last_window_failures: Number of failures in the previous 5-minute window (None if no previous window) + sent: Whether an alert has been sent for this issue + last_alert: ISO format datetime string of when the last alert was sent (None if never sent) + """ + + type: str + window_start: str + failures: int + last_window_failures: Optional[int] + sent: bool + last_alert: Optional[str] + + class Dispatch: """ Load configuration for each check/state pair, run the check, and run @@ -35,7 +57,7 @@ def __init__( ) -> None: self.config = config self.publishers = publishers - self.open_alerts: Dict[str, Any] = {} + self.open_alerts: Dict[str, AlertInfo] = {} if "ZendutyEvent" in self.config["events"]: self.open_alerts_file = os.environ["OPEN_ALERTS_FILE"] self.open_alerts = self.load_alerts() @@ -43,10 +65,17 @@ def __init__( # events cannot be stored in open_alerts as they are not JSON serializable. self.delayed_events: Dict[str, Event] = {} - def load_alerts(self) -> Dict[str, Any]: + def load_alerts(self) -> Dict[str, AlertInfo]: try: with open(self.open_alerts_file, "r") as file: - return json.load(file) + loaded = json.load(file) + # Ensure all required fields are present + for alert_id, alert in loaded.items(): + if "last_window_failures" not in alert: + alert["last_window_failures"] = None + if "last_alert" not in alert: + alert["last_alert"] = None + return loaded # type: ignore[return-value] except FileNotFoundError: return {} # Return an empty dict if the file doesn't exist @@ -74,7 +103,7 @@ async def run(self, states: List[State]) -> None: event: Event = globals()[event_type](check, context) if event_type in ["ZendutyEvent", "TelegramEvent"]: - alert_identifier = self.generate_alert_identifier(check) + alert_identifier = generate_alert_identifier(check) alert = self.open_alerts.get(alert_identifier) if alert is None: self.open_alerts[alert_identifier] = { @@ -83,9 +112,14 @@ async def run(self, states: List[State]) -> None: "failures": 1, "last_window_failures": None, "sent": False, + "last_alert": None, } else: + # Check window status before incrementing to avoid losing current run's failures + self.check_zd_alert_status(alert_identifier, current_time) alert["failures"] += 1 + # Always update delayed_events with the latest event to ensure we send + # the most recent error information when the alert is finally sent self.delayed_events[f"{event_type}-{alert_identifier}"] = event continue # Skip sending immediately for ZendutyEvent or TelegramEvent @@ -168,12 +202,6 @@ def load_config(self, check_name: str, symbol: str) -> Dict[str, Any]: return config # Zenduty Functions - def generate_alert_identifier(self, check: Check) -> str: - alert_identifier = f"{check.__class__.__name__}-{check.state().symbol}" - state = check.state() - if isinstance(state, PublisherState): - alert_identifier += f"-{state.publisher_name}" - return alert_identifier def check_zd_alert_status( self, alert_identifier: str, current_time: datetime @@ -193,16 +221,27 @@ async def process_zenduty_events(self, current_time: datetime) -> None: to_alert = [] for identifier, info in self.open_alerts.items(): + # Check window status (idempotent - safe to call multiple times) + # This handles alerts that didn't have failures in the current run self.check_zd_alert_status(identifier, current_time) check_config = self.config["checks"]["global"][info["type"]] alert_threshold = check_config.get("alert_threshold", 5) resolution_threshold = check_config.get("resolution_threshold", 3) - # Resolve the alert if raised and failed < $threshold times in the last 5m window + # Resolve the alert if raised and failed <= $threshold times in the last 5m window + # OR if the current window has low failures (for immediate resolution) resolved = False - if ( + # Check if last window had low failures + last_window_resolved = ( info["last_window_failures"] is not None and info["last_window_failures"] <= resolution_threshold - ): + ) + # Check if current window has low failures (and alert was previously sent) + current_window_resolved = ( + info["sent"] + and info["failures"] <= resolution_threshold + and info["failures"] < alert_threshold + ) + if last_window_resolved or current_window_resolved: logger.debug(f"Resolving Zenduty alert {identifier}") resolved = True @@ -223,7 +262,7 @@ async def process_zenduty_events(self, current_time: datetime) -> None: elif ( info["failures"] >= alert_threshold or (info["sent"] and not resolved) ) and ( - not info.get("last_alert") # First alert - send immediately + info["last_alert"] is None # First alert - send immediately or ( # Subsequent alerts - send at the start of each hour current_time - datetime.fromisoformat(info["last_alert"]) > timedelta(minutes=5) @@ -233,15 +272,17 @@ async def process_zenduty_events(self, current_time: datetime) -> None: logger.debug(f"Raising Zenduty alert {identifier}") self.open_alerts[identifier]["sent"] = True self.open_alerts[identifier]["last_alert"] = current_time.isoformat() - for event_type in ["ZendutyEvent", "TelegramEvent"]: - key = f"{event_type}-{identifier}" - event = self.delayed_events.get(key) - if event: - to_alert.append(event.send()) - metrics.alerts_sent_total.labels( - alert_type=info["type"], - channel=event_type.lower().replace("event", ""), - ).inc() + # Only send events for event types that are actually enabled + for event_type in self.config["events"]: + if event_type in ["ZendutyEvent", "TelegramEvent"]: + key = f"{event_type}-{identifier}" + event = self.delayed_events.get(key) + if event: + to_alert.append(event.send()) + metrics.alerts_sent_total.labels( + alert_type=info["type"], + channel=event_type.lower().replace("event", ""), + ).inc() # Send the alerts that were delayed due to thresholds await asyncio.gather(*to_alert) @@ -250,10 +291,12 @@ async def process_zenduty_events(self, current_time: datetime) -> None: for identifier in to_remove: if self.open_alerts.get(identifier): del self.open_alerts[identifier] - for event_type in ["ZendutyEvent", "TelegramEvent"]: - key = f"{event_type}-{identifier}" - if self.delayed_events.get(key): - del self.delayed_events[key] + # Only clean up delayed_events for event types that are actually enabled + for event_type in self.config["events"]: + if event_type in ["ZendutyEvent", "TelegramEvent"]: + key = f"{event_type}-{identifier}" + if self.delayed_events.get(key): + del self.delayed_events[key] metrics.update_alert_metrics(self.open_alerts) diff --git a/pyth_observer/event.py b/pyth_observer/event.py index 2e98d5b..c175cb1 100644 --- a/pyth_observer/event.py +++ b/pyth_observer/event.py @@ -10,6 +10,7 @@ from dotenv import load_dotenv from loguru import logger +from pyth_observer.alert_utils import generate_alert_identifier from pyth_observer.check import Check from pyth_observer.check.publisher import PublisherCheck, PublisherState from pyth_observer.models import Publisher @@ -164,12 +165,9 @@ async def send(self) -> None: for key, value in event_details.items(): summary += f"{key}: {value}\n" - alert_identifier = ( - f"{self.check.__class__.__name__}-{self.check.state().symbol}" - ) + alert_identifier = generate_alert_identifier(self.check) state = self.check.state() if isinstance(state, PublisherState): - alert_identifier += f"-{state.publisher_name}" symbol = ( self.check.state().symbol.replace(".", "-").replace("/", "-").lower() ) diff --git a/pyth_observer/metrics.py b/pyth_observer/metrics.py index 1d04e20..2c22adb 100644 --- a/pyth_observer/metrics.py +++ b/pyth_observer/metrics.py @@ -131,13 +131,6 @@ def __init__(self, registry: CollectorRegistry = REGISTRY): registry=registry, ) - self.crosschain_price_age = Gauge( - "pyth_observer_crosschain_price_age_seconds", - "Age of cross-chain price data in seconds", - ["symbol"], - registry=registry, - ) - self.latest_block_slot = Gauge( "pyth_observer_latest_block_slot", "Latest Solana block slot observed", @@ -205,13 +198,6 @@ def update_price_feed_metrics(self, state: PriceFeedState) -> None: age = time.time() - state.coingecko_update self.coingecko_price_age.labels(symbol=state.symbol).set(age) - if state.crosschain_price and state.crosschain_price.get("publish_time"): - age = ( - state.crosschain_price["snapshot_time"] - - state.crosschain_price["publish_time"] - ) - self.crosschain_price_age.labels(symbol=state.symbol).set(age) - self.latest_block_slot.set(state.latest_block_slot) def record_api_request( diff --git a/sample.coingecko.yaml b/sample.coingecko.yaml index 215ddd8..f3202e0 100644 --- a/sample.coingecko.yaml +++ b/sample.coingecko.yaml @@ -1,60 +1,566 @@ { - "AAVE": { "api": "aave", "market": "aave" }, - "ADA": { "api": "cardano", "market": "cardano" }, - "ALGO": { "api": "algorand", "market": "algorand" }, - "ANC": { "api": "anchor-protocol", "market": "anchor-protocol" }, - "APE": { "api": "apecoin", "market": "apecoin" }, - "ATLAS": { "api": "star-atlas", "market": "star-atlas" }, - "ATOM": { "api": "cosmos", "market": "cosmos" }, - "AUST": { "api": "anchorust", "market": "anchorust" }, - "AVAX": { "api": "avalanche-2", "market": "avalanche" }, - "BCH": { "api": "bitcoin-cash", "market": "bitcoin-cash" }, - "BETH": { "api": "binance-eth", "market": "binance-eth-staking" }, - "BNB": { "api": "binancecoin", "market": "bnb" }, - "BRZ": { "api": "brz", "market": "brazilian-digital-token" }, - "BTC": { "api": "bitcoin", "market": "bitcoin" }, - "BUSD": { "api": "binance-usd", "market": "binance-usd" }, - "C98": { "api": "coin98", "market": "coin98" }, - "COPE": { "api": "cope", "market": "cope" }, - "CUSD": { "api": "celo-dollar", "market": "celo-dollar" }, - "DOGE": { "api": "dogecoin", "market": "dogecoin" }, - "DOT": { "api": "polkadot", "market": "polkadot" }, - "ETH": { "api": "ethereum", "market": "ethereum" }, - "FIDA": { "api": "bonfida", "market": "bonfida" }, - "FTM": { "api": "fantom", "market": "fantom" }, - "FTT": { "api": "ftx-token", "market": "ftx-token" }, - "GMT": { "api": "stepn", "market": "stepn" }, - "GOFX": { "api": "goosefx", "market": "goosefx" }, - "HXRO": { "api": "hxro", "market": "hxro" }, - "INJ": { "api": "injective-protocol", "market": "injective" }, - "JET": { "api": "jet", "market": "jet" }, - "LTC": { "api": "litecoin", "market": "litecoin" }, - "LUNA": { "api": "terra-luna-2", "market": "terra" }, - "LUNC": { "api": "terra-luna", "market": "terra-luna-classic" }, - "MATIC": { "api": "matic-network", "market": "polygon" }, - "MER": { "api": "mercurial", "market": "mercurial" }, - "MIR": { "api": "mirror-protocol", "market": "mirror-protocol" }, - "MNGO": { "api": "mango-markets", "market": "mango" }, - "MSOL": { "api": "msol", "market": "marinade-staked-sol" }, - "NEAR": { "api": "near", "market": "near" }, - "ONE": { "api": "harmony", "market": "harmony" }, - "ORCA": { "api": "orca", "market": "orca" }, - "PAI": { "api": "parrot-usd", "market": "parrot-usd" }, - "PORT": { "api": "port-finance", "market": "port-finance" }, - "RAY": { "api": "raydium", "market": "raydium" }, - "SBR": { "api": "saber", "market": "saber" }, - "SCNSOL": { "api": "socean-staked-sol", "market": "socean-staked-sol" }, - "SLND": { "api": "solend", "market": "solend" }, - "SNY": { "api": "synthetify-token", "market": "synthetify-token" }, - "SOL": { "api": "solana", "market": "solana" }, - "SRM": { "api": "serum", "market": "serum" }, - "STEP": { "api": "step-finance", "market": "step-finance" }, - "STSOL": { "api": "lido-staked-sol", "market": "lido-staked-sol" }, - "TUSD": { "api": "true-usd", "market": "true-usd" }, - "USDC": { "api": "usd-coin", "market": "usd-coin" }, - "USDT": { "api": "tether", "market": "tether" }, - "USTC": { "api": "terrausd", "market": "terraclassicusd" }, - "VAI": { "api": "vai", "market": "vai" }, - "XVS": { "api": "venus", "market": "venus" }, - "ZBC": { "api": "zebec-protocol", "market": "zebec-protocol" }, -} + "Crypto.0G/USD": "zero-gravity", + "Crypto.1INCH/USD": "1inch", + "Crypto.2Z/USD": "doublezero", + "Crypto.4/USD": "4-2", + "Crypto.A/USD": "vaulta", + "Crypto.AAPLX/USD": "apple-xstock", + "Crypto.AAVE/USD": "aave", + "Crypto.ACT/USD": "act-i-the-ai-prophecy", + "Crypto.ADA/USD": "cardano", + "Crypto.AERGO/USD": "aergo", + "Crypto.AERO/USD": "aerodrome-finance", + "Crypto.AEVO/USD": "aevo-exchange", + "Crypto.AFSUI/USD": "aftermath-staked-sui", + "Crypto.AI16Z/USD": "ai16z", + "Crypto.AIXBT/USD": "aixbt", + "Crypto.AKT/USD": "akash-network", + "Crypto.ALGO/USD": "algorand", + "Crypto.ALICE/USD": "alice", + "Crypto.ALKIMI/USD": "alkimi-2", + "Crypto.ALT/USD": "altlayer", + "Crypto.AMI/USD": "ami", + "Crypto.AMP/USD": "amp-token", + "Crypto.ANIME/USD": "anime", + "Crypto.ANKR/USD": "ankr", + "Crypto.ANON/USD": "anon-2", + "Crypto.APE/USD": "apecoin", + "Crypto.APEX/USD": "apex-token-2", + "Crypto.API3/USD": "api3", + "Crypto.APT/USD": "aptos", + "Crypto.AR/USD": "arweave", + "Crypto.ARB/USD": "arbitrum", + "Crypto.ARC/USD": "ai-rig-complex", + "Crypto.ARKM/USD": "arkham", + "Crypto.ASTER/USD": "aster-2", + "Crypto.ASTR/USD": "astar", + "Crypto.ATH/USD": "aethir", + "Crypto.ATLAS/USD": "star-atlas", + "Crypto.ATOM/USD": "cosmos", + "Crypto.AUDD/USD": "novatti-australian-digital-dollar", + "Crypto.AUDIO/USD": "audius", + "Crypto.AURORA/USD": "aurora-near", + "Crypto.AUSD/USD": "agora-dollar", + "Crypto.AVAIL/USD": "avail", + "Crypto.AVAX/USD": "avalanche-2", + "Crypto.AVNT/USD": "avantis", + "Crypto.AXL/USD": "axelar", + "Crypto.AXS/USD": "axie-infinity", + "Crypto.B3/USD": "b3", + "Crypto.BABY/USD": "babylon", + "Crypto.BABYDOGE/USD": "baby-doge-coin", + "Crypto.BAL/USD": "balancer", + "Crypto.BAN/USD": "comedian", + "Crypto.BAND/USD": "band-protocol", + "Crypto.BAT/USD": "basic-attention-token", + "Crypto.BBSOL/USD": "bybit-staked-sol", + "Crypto.BCH/USD": "bitcoin-cash", + "Crypto.BELIEVE/USD": "ben-pasternak", + "Crypto.BENJI/USD": "basenji", + "Crypto.BERA/USD": "berachain-bera", + "Crypto.BGB/USD": "bitget-token", + "Crypto.BIO/USD": "bio-protocol", + "Crypto.BITCOIN/USD": "harrypotterobamasonic10inu", + "Crypto.BLAST/USD": "blast", + "Crypto.BLUE/USD": "bluefin", + "Crypto.BLUR/USD": "blur", + "Crypto.BLZE/USD": "solblaze", + "Crypto.BMT/USD": "bubblemaps", + "Crypto.BNB/USD": "binancecoin", + "Crypto.BNSOL/USD": "binance-staked-sol", + "Crypto.BOBA/USD": "boba-network", + "Crypto.BODEN/USD": "jeo-boden", + "Crypto.BOLD/USD": "bold-2", + "Crypto.BOME/USD": "book-of-meme", + "Crypto.BONK/USD": "bonk", + "Crypto.BORG/USD": "swissborg", + "Crypto.BRETT/USD": "brett", + "Crypto.BROCCOLI/USD": "czs-dog", + "Crypto.BSOL/USD": "blazestake-staked-sol", + "Crypto.BSV/USD": "bitcoin-cash-sv", + "Crypto.BTC/USD": "bitcoin", + "Crypto.BTT/USD": "bittorrent", + "Crypto.BUCK/USD": "bucket-protocol-buck-stablecoin", + "Crypto.BUCKET.USDB/USD": "bucket-usd", + "Crypto.BUDDY/USD": "alright-buddy", + "Crypto.BYUSD/USD": "byusd", + "Crypto.C98/USD": "coin98", + "Crypto.CAKE/USD": "pancakeswap-token", + "Crypto.CAMP/USD": "camp-network", + "Crypto.CARV/USD": "carv", + "Crypto.CASH/USD": "cash-2", + "Crypto.CAT/USD": "cat-3", + "Crypto.CBBTC/USD": "coinbase-wrapped-btc", + "Crypto.CBDOGE/USD": "coinbase-wrapped-doge", + "Crypto.CBETH/USD": "coinbase-wrapped-staked-eth", + "Crypto.CBXRP/USD": "coinbase-wrapped-xrp", + "Crypto.CC/USD": "canton-network", + "Crypto.CELO/USD": "celo", + "Crypto.CELR/USD": "celer-network", + "Crypto.CETUS/USD": "cetus-protocol", + "Crypto.CFX/USD": "conflux-token", + "Crypto.CHILLGUY/USD": "chill-guy", + "Crypto.CHR/USD": "chromaway", + "Crypto.CHZ/USD": "chiliz", + "Crypto.CLANKER/USD": "tokenbot-2", + "Crypto.CLOUD/USD": "sanctum-2", + "Crypto.COINX/USD": "coinbase-xstock", + "Crypto.COMP/USD": "compound-governance-token", + "Crypto.COOK/USD": "meth-protocol", + "Crypto.COOKIE/USD": "cookie", + "Crypto.COQ/USD": "coq-inu", + "Crypto.CORE/USD": "core-2", + "Crypto.COW/USD": "cow", + "Crypto.CRCLX/USD": "circle-xstock", + "Crypto.CRO/USD": "crypto-com-chain", + "Crypto.CRV/USD": "curve-dao-token", + "Crypto.CSPR/USD": "casper-network", + "Crypto.CTSI/USD": "cartesi", + "Crypto.CVX/USD": "convex-finance", + "Crypto.DAI/USD": "dai", + "Crypto.DASH/USD": "dash", + "Crypto.DBR/USD": "debridge", + "Crypto.DEEP/USD": "deep", + "Crypto.DEGEN/USD": "degen-base", + "Crypto.DEUSD/USD": "elixir-deusd", + "Crypto.DEXE/USD": "dexe", + "Crypto.DMC/USD": "delorean", + "Crypto.DODO/USD": "dodo", + "Crypto.DOGE/USD": "dogecoin", + "Crypto.DOGINME/USD": "doginme", + "Crypto.DOGS/USD": "dogs-2", + "Crypto.DOLO/USD": "dolomite", + "Crypto.DOT/USD": "polkadot", + "Crypto.DRIFT/USD": "drift-protocol", + "Crypto.DSOL/USD": "drift-staked-sol", + "Crypto.DYDX/USD": "dydx-chain", + "Crypto.DYM/USD": "dymension", + "Crypto.EBTC/USD": "ether-fi-staked-btc", + "Crypto.ECHO/USD": "echo-protocol", + "Crypto.EDU/USD": "edu-coin", + "Crypto.EGLD/USD": "elrond-erd-2", + "Crypto.EIGEN/USD": "eigenlayer", + "Crypto.ELIZAOS/USD": "elizaos", + "Crypto.ELON/USD": "dogelon-mars", + "Crypto.ENA/USD": "ethena", + "Crypto.ENJ/USD": "enjincoin", + "Crypto.ENS/USD": "ethereum-name-service", + "Crypto.ES/USD": "eclipse-3", + "Crypto.ETC/USD": "ethereum-classic", + "Crypto.ETH/USD": "ethereum", + "Crypto.ETHFI/USD": "ether-fi", + "Crypto.ETHW/USD": "ethereum-pow-iou", + "Crypto.EUL/USD": "euler", + "Crypto.EURC/USD": "euro-coin", + "Crypto.EURCV/USD": "societe-generale-forge-eurcv", + "Crypto.EVAA/USD": "evaa-protocol", + "Crypto.EZETH/USD": "renzo-restaked-eth", + "Crypto.F/USD": "synfutures", + "Crypto.FAI/USD": "freysa-ai", + "Crypto.FARTCOIN/USD": "fartcoin", + "Crypto.FDIT/USD": "fidelity-digital-interest-token", + "Crypto.FDUSD/USD": "first-digital-usd", + "Crypto.FET/USD": "fetch-ai", + "Crypto.FEUSD/USD": "felix-feusd", + "Crypto.FF/USD": "falcon-finance-ff", + "Crypto.FIDA/USD": "bonfida", + "Crypto.FIL/USD": "filecoin", + "Crypto.FLOKI/USD": "floki", + "Crypto.FLOW/USD": "flow", + "Crypto.FLR/USD": "flare-networks", + "Crypto.FLUID/USD": "instadapp", + "Crypto.FORM/USD": "four", + "Crypto.FOXY/USD": "foxy", + "Crypto.FRAG/USD": "fragmetric", + "Crypto.FRAX/USD": "frax", + "Crypto.FRXETH/USD": "frax-ether", + "Crypto.FRXUSD/USD": "frax-usd", + "Crypto.FTT/USD": "ftx-token", + "Crypto.FUEL/USD": "fuel-network", + "Crypto.FWOG/USD": "fwog", + "Crypto.G/USD": "g-2", + "Crypto.GALA/USD": "gala", + "Crypto.GHO/USD": "gho", + "Crypto.GIGA/USD": "gigachad-2", + "Crypto.GLM/USD": "golem", + "Crypto.GLMR/USD": "moonbeam", + "Crypto.GMT/USD": "stepn", + "Crypto.GMX/USD": "gmx", + "Crypto.GNO/USD": "gnosis", + "Crypto.GNS/USD": "gains-network", + "Crypto.GOAT/USD": "goat", + "Crypto.GOGLZ/USD": "googles", + "Crypto.GOLD/USD": "gold-2", + "Crypto.GOOGLX/USD": "alphabet-xstock", + "Crypto.GORK/USD": "gork", + "Crypto.GP/USD": "graphite-protocol", + "Crypto.GPS/USD": "goplus-security", + "Crypto.GRAIL/USD": "camelot-token", + "Crypto.GRASS/USD": "grass", + "Crypto.GRIFFAIN/USD": "griffain", + "Crypto.GRT/USD": "the-graph", + "Crypto.GT/USD": "gatechain-token", + "Crypto.GUSD/USD": "gemini-dollar", + "Crypto.H/USD": "humanity", + "Crypto.HAEDAL/USD": "haedal", + "Crypto.HASUI/USD": "haedal-staked-sui", + "Crypto.HBAR/USD": "hedera-hashgraph", + "Crypto.HEMI/USD": "hemi", + "Crypto.HFT/USD": "hashflow", + "Crypto.HFUN/USD": "hypurr-fun", + "Crypto.HIPPO/USD": "sudeng", + "Crypto.HNT/USD": "helium", + "Crypto.HOLO/USD": "holoworld", + "Crypto.HONEY/USD": "honey-3", + "Crypto.HOODX/USD": "robinhood-xstock", + "Crypto.HT/USD": "huobi-token", + "Crypto.HUMA/USD": "huma-finance", + "Crypto.HYPE/USD": "hyperliquid", + "Crypto.HYPER/USD": "hyperlane", + "Crypto.HYPERSTABLE.USH/USD": "hyperstable", + "Crypto.IBERA/USD": "infrared-bera", + "Crypto.IBGT/USD": "infrafred-bgt", + "Crypto.ICP/USD": "internet-computer", + "Crypto.ICX/USD": "icon", + "Crypto.IDEX/USD": "aurora-dao", + "Crypto.IKA/USD": "ika", + "Crypto.ILV/USD": "illuvium", + "Crypto.IMX/USD": "immutable-x", + "Crypto.INF/USD": "socean-staked-sol", + "Crypto.INIT/USD": "initia", + "Crypto.INJ/USD": "injective-protocol", + "Crypto.IO/USD": "io", + "Crypto.IOTA/USD": "iota", + "Crypto.IOTX/USD": "iotex", + "Crypto.IP/USD": "story-2", + "Crypto.JASMY/USD": "jasmycoin", + "Crypto.JITOSOL/USD": "jito-staked-sol", + "Crypto.JLP/USD": "jupiter-perpetuals-liquidity-provider-token", + "Crypto.JOE/USD": "joe", + "Crypto.JTO/USD": "jito-governance-token", + "Crypto.JUP/USD": "jupiter", + "Crypto.KAIA/USD": "kaia", + "Crypto.KAITO/USD": "kaito", + "Crypto.KAPT/USD": "kofi-aptos", + "Crypto.KAS/USD": "kaspa", + "Crypto.KAVA/USD": "kava", + "Crypto.KCS/USD": "kucoin-shares", + "Crypto.KERNEL/USD": "kernel-2", + "Crypto.KHYPE/USD": "kinetic-staked-hype", + "Crypto.KMNO/USD": "kamino", + "Crypto.KNC/USD": "kyber-network-crystal", + "Crypto.KNTQ/USD": "kinetiq", + "Crypto.KSM/USD": "kusama", + "Crypto.KTA/USD": "keeta", + "Crypto.LA/USD": "lagrange", + "Crypto.LAYER/USD": "solayer", + "Crypto.LBGT/USD": "liquid-bgt", + "Crypto.LBTC/USD": "lombard-staked-btc", + "Crypto.LDO/USD": "lido-dao", + "Crypto.LEO/USD": "leo-2", + "Crypto.LHYPE/USD": "looped-hype", + "Crypto.LINEA/USD": "linea", + "Crypto.LINK/USD": "chainlink", + "Crypto.LION/USD": "loaded-lions", + "Crypto.LL/USD": "lightlink", + "Crypto.LMTS/USD": "limitless-3", + "Crypto.LOFI/USD": "lofi-2", + "Crypto.LOOKS/USD": "looksrare", + "Crypto.LQTY/USD": "liquity", + "Crypto.LRC/USD": "loopring", + "Crypto.LST/USD": "liquid-staking-token", + "Crypto.LTC/USD": "litecoin", + "Crypto.LUCE/USD": "offcial-mascot-of-the-holy-year", + "Crypto.LUNA/USD": "terra-luna-2", + "Crypto.LUNC/USD": "terra-luna", + "Crypto.LUSD/USD": "liquity-usd", + "Crypto.MAG7-SSI/USD": "mag7-ssi", + "Crypto.MANA/USD": "decentraland", + "Crypto.MANEKI/USD": "maneki", + "Crypto.MANTA/USD": "manta-network", + "Crypto.MASK/USD": "mask-network", + "Crypto.MAV/USD": "maverick-protocol", + "Crypto.MCDX/USD": "mcdonald-s-xstock", + "Crypto.ME/USD": "magic-eden", + "Crypto.MELANIA/USD": "melania-meme", + "Crypto.MEME/USD": "memecoin-2", + "Crypto.MERL/USD": "merlin-chain", + "Crypto.MET/USD": "meteora", + "Crypto.META/USD": "meta-2-2", + "Crypto.METASTABLE.MUSD/USD": "mad-usd", + "Crypto.METAX/USD": "meta-xstock", + "Crypto.METH/USD": "mantle-staked-ether", + "Crypto.METIS/USD": "metis-token", + "Crypto.MEW/USD": "cat-in-a-dogs-world", + "Crypto.MEZO.MUSD/USD": "mezo-usd", + "Crypto.MHYPE/USD": "hyperpie-staked-mhype", + "Crypto.MIM/USD": "magic-internet-money", + "Crypto.MINA/USD": "mina-protocol", + "Crypto.MMT/USD": "momentum-3", + "Crypto.MNDE/USD": "marinade", + "Crypto.MNT/USD": "mantle", + "Crypto.MOBILE/USD": "helium-mobile", + "Crypto.MOBY/USD": "moby", + "Crypto.MODE/USD": "mode", + "Crypto.MOG/USD": "mog", + "Crypto.MON/USD": "monad", + "Crypto.MOODENG/USD": "moo-deng", + "Crypto.MORPHO/USD": "morpho", + "Crypto.MOTHER/USD": "mother-iggy", + "Crypto.MOVE/USD": "movement", + "Crypto.MSETH/USD": "metronome-synth-eth", + "Crypto.MSOL/USD": "msol", + "Crypto.MSTRX/USD": "microstrategy-xstock", + "Crypto.MSUSD/USD": "main-street-usd", + "Crypto.MTRG/USD": "meter", + "Crypto.MUBARAK/USD": "mubarak", + "Crypto.MYRO/USD": "myro", + "Crypto.MYX/USD": "myx-finance", + "Crypto.NAVX/USD": "navi", + "Crypto.NEAR/USD": "near", + "Crypto.NECT/USD": "nectar", + "Crypto.NEIRO/USD": "neiro", + "Crypto.NEON/USD": "neon", + "Crypto.NEXO/USD": "nexo", + "Crypto.NFLXX/USD": "netflix-xstock", + "Crypto.NIL/USD": "nillion", + "Crypto.NOBODY/USD": "nobody-sausage", + "Crypto.NOT/USD": "notcoin", + "Crypto.NS/USD": "nodestats", + "Crypto.NTRN/USD": "neutron-3", + "Crypto.NVDAX/USD": "nvidia-xstock", + "Crypto.NXPC/USD": "nexpace", + "Crypto.ODOS/USD": "odos", + "Crypto.OG/USD": "og-fan-token", + "Crypto.OGN/USD": "origin-protocol", + "Crypto.OHM/USD": "olympus", + "Crypto.OKB/USD": "okb", + "Crypto.OM/USD": "mantra-dao", + "Crypto.OMI/USD": "ecomi", + "Crypto.ONDO/USD": "ondo-finance", + "Crypto.ONE/USD": "harmony", + "Crypto.OP/USD": "optimism", + "Crypto.ORCA/USD": "orca", + "Crypto.ORDER/USD": "order-2", + "Crypto.ORDI/USD": "ordinals", + "Crypto.ORE/USD": "ore", + "Crypto.OS/USD": "origin-staked-s", + "Crypto.OSMO/USD": "osmosis", + "Crypto.OUSDT/USD": "openusdt", + "Crypto.P33/USD": "pharaoh-liquid-staking-token", + "Crypto.PARTI/USD": "particle-network", + "Crypto.PAXG/USD": "pax-gold", + "Crypto.PENDLE/USD": "pendle", + "Crypto.PENGU/USD": "penguiana", + "Crypto.PEOPLE/USD": "constitutiondao", + "Crypto.PEPE/USD": "pepe", + "Crypto.PERP/USD": "perpetual-protocol", + "Crypto.PI/USD": "pi-network", + "Crypto.PLUME/USD": "plume", + "Crypto.PNUT/USD": "peanut-3", + "Crypto.POL/USD": "polygon-ecosystem-token", + "Crypto.PONKE/USD": "ponke", + "Crypto.POPCAT/USD": "popcat", + "Crypto.PRCL/USD": "parcl", + "Crypto.PRIME/USD": "echelon-prime", + "Crypto.PROMPT/USD": "wayfinder", + "Crypto.PROVE/USD": "succinct", + "Crypto.PSG/USD": "paris-saint-germain-fan-token", + "Crypto.PUMP/USD": "pump", + "Crypto.PURR/USD": "purr-2", + "Crypto.PYTH/USD": "pyth-network", + "Crypto.PYUSD/USD": "paypal-usd", + "Crypto.QNT/USD": "quant-network", + "Crypto.QQQX/USD": "nasdaq-xstock", + "Crypto.QTUM/USD": "qtum", + "Crypto.RAY/USD": "raydium", + "Crypto.RDNT/USD": "radiant-capital", + "Crypto.RED/USD": "redstone-oracles", + "Crypto.RENDER/USD": "render-token", + "Crypto.RESOLV/USD": "resolv", + "Crypto.RETARDIO/USD": "retardio", + "Crypto.RETH/USD": "rocket-pool-eth", + "Crypto.REX33/USD": "etherex-liquid-staking-token", + "Crypto.REZ/USD": "renzo", + "Crypto.RHEA/USD": "rhea-2", + "Crypto.RION/USD": "hyperion-2", + "Crypto.RLB/USD": "rollbit-coin", + "Crypto.RLP/USD": "resolv-rlp", + "Crypto.RLUSD/USD": "ripple-usd", + "Crypto.RON/USD": "ronin", + "Crypto.ROSE/USD": "oasis-network", + "Crypto.RPL/USD": "rocket-pool", + "Crypto.RSETH/USD": "kelp-dao-restaked-eth", + "Crypto.RSR/USD": "reserve-rights-token", + "Crypto.RSWETH/USD": "restaked-swell-eth", + "Crypto.RUNE/USD": "thorchain", + "Crypto.S/USD": "sonic-3", + "Crypto.SAFE/USD": "safe", + "Crypto.SAMO/USD": "samoyedcoin", + "Crypto.SAND/USD": "the-sandbox", + "Crypto.SATS/USD": "sats-ordinals", + "Crypto.SCA/USD": "scallop-2", + "Crypto.SCETH/USD": "rings-sc-eth", + "Crypto.SCR/USD": "scroll", + "Crypto.SCRT/USD": "secret", + "Crypto.SCUSD/USD": "rings-scusd", + "Crypto.SD/USD": "stader", + "Crypto.SDAI/USD": "savings-dai", + "Crypto.SEDA/USD": "seda-2", + "Crypto.SEI/USD": "sei-network", + "Crypto.SEND/USD": "suilend", + "Crypto.SFRXETH/USD": "staked-frax-ether", + "Crypto.SHADOW/USD": "shadow-2", + "Crypto.SHIB/USD": "shiba-inu", + "Crypto.SIGN/USD": "sign-global", + "Crypto.SKATE/USD": "skate", + "Crypto.SKI/USD": "ski-mask-dog", + "Crypto.SKL/USD": "skale", + "Crypto.SKY/USD": "sky", + "Crypto.SLP/USD": "smooth-love-potion", + "Crypto.SNX/USD": "havven", + "Crypto.SOL/USD": "solana", + "Crypto.SOLV/USD": "solv-protocol", + "Crypto.SOLVBTC/USD": "solv-btc", + "Crypto.SONIC/USD": "sonic-2", + "Crypto.SOON/USD": "soon-2", + "Crypto.SOPH/USD": "sophon", + "Crypto.SPELL/USD": "spell-token", + "Crypto.SPK/USD": "spark-2", + "Crypto.SPX6900/USD": "based-spx6900", + "Crypto.SPYX/USD": "sp500-xstock", + "Crypto.STBL/USD": "stbl", + "Crypto.STETH/USD": "staked-ether", + "Crypto.STG/USD": "stargate-finance", + "Crypto.STHYPE/USD": "staked-hype", + "Crypto.STONE/USD": "stakestone-ether", + "Crypto.STORJ/USD": "storj", + "Crypto.STREAM/USD": "streamflow", + "Crypto.STRK/USD": "starknet", + "Crypto.STS/USD": "beets-staked-sonic", + "Crypto.STSUI/USD": "alphafi-staked-sui", + "Crypto.STX/USD": "blockstack", + "Crypto.SUI/USD": "sui", + "Crypto.SUN/USD": "sun-token", + "Crypto.SUSDE/USD": "ethena-staked-usde", + "Crypto.SUSHI/USD": "sushi", + "Crypto.SWARMS/USD": "swarms", + "Crypto.SWETH/USD": "sweth", + "Crypto.SXP/USD": "swipe", + "Crypto.SYN/USD": "synapse-2", + "Crypto.SYRUP/USD": "syrup", + "Crypto.TAC/USD": "tac", + "Crypto.TAIKO/USD": "taiko", + "Crypto.TAO/USD": "bittensor", + "Crypto.TBTC/USD": "tbtc", + "Crypto.THAPT/USD": "thala-apt", + "Crypto.THE/USD": "thena", + "Crypto.THETA/USD": "theta-token", + "Crypto.THL/USD": "thala", + "Crypto.TIA/USD": "celestia", + "Crypto.TNSR/USD": "tensor", + "Crypto.TOKEN/USD": "tokenfi", + "Crypto.TON/USD": "the-open-network", + "Crypto.TOSHI/USD": "toshi", + "Crypto.TRB/USD": "tellor", + "Crypto.TRUMP/USD": "official-trump", + "Crypto.TRX/USD": "tron", + "Crypto.TSLAX/USD": "tesla-xstock", + "Crypto.TST/USD": "test-3", + "Crypto.TURBO/USD": "turbo", + "Crypto.TURBOS/USD": "turbos-finance", + "Crypto.TUSD/USD": "true-usd", + "Crypto.TUT/USD": "tutorial", + "Crypto.TWT/USD": "trust-wallet-token", + "Crypto.UBTC/USD": "unit-bitcoin", + "Crypto.UETH/USD": "unit-ethereum", + "Crypto.UFART/USD": "unit-fartcoin", + "Crypto.UMA/USD": "uma", + "Crypto.UNI/USD": "uniswap", + "Crypto.UP/USD": "doubleup", + "Crypto.URANUS/USD": "uranus-2", + "Crypto.USD0++/USD": "usd0-liquid-bond", + "Crypto.USD0/USD": "usual-usd", + "Crypto.USDA/USD": "auro-usda", + "Crypto.USDAF/USD": "asymmetry-usdaf-2", + "Crypto.USDAI/USD": "usdai", + "Crypto.USDB/USD": "bucket-usd", + "Crypto.USDC/USD": "usd-coin", + "Crypto.USDD/USD": "usdd", + "Crypto.USDE/USD": "ethena-usde", + "Crypto.USDF/USD": "falcon-finance", + "Crypto.USDG/USD": "global-dollar", + "Crypto.USDH/USD": "hermetica-usdh", + "Crypto.USDHL/USD": "hyper-usd", + "Crypto.USDL/USD": "lift-dollar", + "Crypto.USDN/USD": "smardex-usdn", + "Crypto.USDP/USD": "paxos-standard", + "Crypto.USDS/USD": "usds", + "Crypto.USDT/USD": "tether", + "Crypto.USDT0/USD": "usdt0", + "Crypto.USDTB/USD": "usdtb", + "Crypto.USDU/USD": "uncap-usd", + "Crypto.USDXL/USD": "last-usd", + "Crypto.USDY/USD": "ondo-us-dollar-yield", + "Crypto.USELESS/USD": "useless-3", + "Crypto.USOL/USD": "unit-solana", + "Crypto.USR/USD": "resolv-usr", + "Crypto.USTC/USD": "terrausd", + "Crypto.USUAL/USD": "usual", + "Crypto.USX/USD": "token-dforce-usd", + "Crypto.VANA/USD": "vana", + "Crypto.VELODROME.VELO/USD": "velo", + "Crypto.VET/USD": "vechain", + "Crypto.VIC/USD": "tomochain", + "Crypto.VINE/USD": "vine", + "Crypto.VIRTUAL/USD": "virtual-protocol", + "Crypto.VSUI/USD": "volo-staked-sui", + "Crypto.VVV/USD": "venice-token", + "Crypto.W/USD": "w", + "Crypto.WAGMI/USD": "wagmi-2", + "Crypto.WAL/USD": "walrus-2", + "Crypto.WAVES/USD": "waves", + "Crypto.WBETH/USD": "wrapped-beacon-eth", + "Crypto.WBTC/USD": "wrapped-bitcoin", + "Crypto.WCT/USD": "connect-token-wct", + "Crypto.WEETH/USD": "wrapped-eeth", + "Crypto.WELL/USD": "moonwell-artemis", + "Crypto.WEN/USD": "wen-4", + "Crypto.WETH/USD": "weth", + "Crypto.WFRAGSOL/USD": "wrapped-fragsol", + "Crypto.WIF/USD": "dogwifcoin", + "Crypto.WLD/USD": "worldcoin-wld", + "Crypto.WLFI/USD": "world-liberty-financial", + "Crypto.WOJAK/USD": "wojak", + "Crypto.WOM/USD": "wombat-exchange", + "Crypto.WOO/USD": "woo-network", + "Crypto.WSTETH/USD": "wrapped-steth", + "Crypto.XAI/USD": "xai-blockchain", + "Crypto.XAUT/USD": "tether-gold", + "Crypto.XBTC/USD": "okx-wrapped-btc", + "Crypto.XDC/USD": "xdce-crowd-sale", + "Crypto.XEC/USD": "ecash", + "Crypto.XION/USD": "xion-2", + "Crypto.XLM/USD": "stellar", + "Crypto.XMR/USD": "monero", + "Crypto.XPL/USD": "plasma", + "Crypto.XPRT/USD": "persistence", + "Crypto.XRD/USD": "radix", + "Crypto.XRP/USD": "ripple", + "Crypto.XSGD/USD": "xsgd", + "Crypto.XTZ/USD": "tezos", + "Crypto.YFI/USD": "yearn-finance", + "Crypto.YU/USD": "yu", + "Crypto.YZY/USD": "swasticoin", + "Crypto.ZBTC/USD": "zeus-netwok-zbtc", + "Crypto.ZEC/USD": "zcash", + "Crypto.ZEN/USD": "zencash", + "Crypto.ZEREBRO/USD": "zerebro", + "Crypto.ZETA/USD": "zetachain", + "Crypto.ZEUS/USD": "zeus-2", + "Crypto.ZEX/USD": "zeta", + "Crypto.ZIL/USD": "zilliqa", + "Crypto.ZK/USD": "zksync", + "Crypto.ZORA/USD": "zora", + "Crypto.ZRO/USD": "layerzero" +} \ No newline at end of file diff --git a/sample.config.yaml b/sample.config.yaml index e87e84f..ce159dc 100644 --- a/sample.config.yaml +++ b/sample.config.yaml @@ -3,7 +3,6 @@ network: http_endpoint: "https://api2.pythnet.pyth.network" ws_endpoint: "wss://api2.pythnet.pyth.network" first_mapping: "AHtgzX45WTKfkPG53L6WYhGEXwQkN1BVknET3sVsLL8J" - crosschain_endpoint: "https://hermes.pyth.network" request_rate_limit: 10 request_rate_period: 1 events: @@ -28,13 +27,8 @@ checks: enable: true max_deviation: 5 max_staleness: 60 - PriceFeedCrossChainOnlineCheck: - enable: true - max_staleness: 60 - PriceFeedCrossChainDeviationCheck: - enable: true - max_deviation: 5 - max_staleness: 60 + PriceFeedConfidenceIntervalCheck: + enable: false # Publisher checks PublisherWithinAggregateConfidenceCheck: enable: false diff --git a/scripts/build_coingecko_mapping.py b/scripts/build_coingecko_mapping.py new file mode 100755 index 0000000..0cfa5fe --- /dev/null +++ b/scripts/build_coingecko_mapping.py @@ -0,0 +1,852 @@ +#!/usr/bin/env python3 +""" +Script to build CoinGecko mapping file from Pyth Hermes API and CoinGecko API. + +This script: +1. Fetches all price feeds from Pyth Hermes API +2. Extracts base symbols (especially for Crypto assets) +3. Gets CoinGecko coin list +4. Matches using Pyth description (most reliable) and symbol matching +5. Generates the mapping file with warnings for non-100% matches +""" + +import json +import sys +import time +from difflib import SequenceMatcher +from typing import Any, Dict, List, Optional, Tuple + +import requests +from loguru import logger +from pycoingecko import CoinGeckoAPI + +# Configure logger +logger.remove() +logger.add( + sys.stderr, + level="INFO", + format="{time:YYYY-MM-DD HH:mm:ss} | {level: <8} | {message}", +) + +HERMES_API_URL = "https://hermes.pyth.network/v2/price_feeds" +COINGECKO_API = CoinGeckoAPI() + +# Known mappings for validation only (not used in matching logic) +# Format: Pyth symbol -> CoinGecko ID +KNOWN_MAPPINGS = { + "Crypto.BTC/USD": "bitcoin", + "Crypto.ETH/USD": "ethereum", + "Crypto.USDT/USD": "tether", + "Crypto.USDC/USD": "usd-coin", + "Crypto.BNB/USD": "binancecoin", + "Crypto.SOL/USD": "solana", + "Crypto.XRP/USD": "ripple", + "Crypto.DOGE/USD": "dogecoin", + "Crypto.ADA/USD": "cardano", + "Crypto.AVAX/USD": "avalanche-2", + "Crypto.DOT/USD": "polkadot", + "Crypto.MATIC/USD": "matic-network", + "Crypto.LINK/USD": "chainlink", + "Crypto.UNI/USD": "uniswap", + "Crypto.ATOM/USD": "cosmos", + "Crypto.LTC/USD": "litecoin", + "Crypto.BCH/USD": "bitcoin-cash", + "Crypto.XLM/USD": "stellar", + "Crypto.ALGO/USD": "algorand", + "Crypto.VET/USD": "vechain", + "Crypto.ICP/USD": "internet-computer", + "Crypto.FIL/USD": "filecoin", + "Crypto.TRX/USD": "tron", + "Crypto.ETC/USD": "ethereum-classic", + "Crypto.EOS/USD": "eos", + "Crypto.AAVE/USD": "aave", + "Crypto.MKR/USD": "maker", + "Crypto.COMP/USD": "compound-governance-token", + "Crypto.YFI/USD": "yearn-finance", + "Crypto.SNX/USD": "havven", + "Crypto.SUSHI/USD": "sushi", + "Crypto.CRV/USD": "curve-dao-token", + "Crypto.1INCH/USD": "1inch", + "Crypto.ENJ/USD": "enjincoin", + "Crypto.BAT/USD": "basic-attention-token", + "Crypto.ZRX/USD": "0x", + "Crypto.MANA/USD": "decentraland", + "Crypto.SAND/USD": "the-sandbox", + "Crypto.GALA/USD": "gala", + "Crypto.AXS/USD": "axie-infinity", + "Crypto.CHZ/USD": "chiliz", + "Crypto.FLOW/USD": "flow", + "Crypto.NEAR/USD": "near", + "Crypto.FTM/USD": "fantom", + "Crypto.HBAR/USD": "hedera-hashgraph", + "Crypto.EGLD/USD": "elrond-erd-2", + "Crypto.THETA/USD": "theta-token", + "Crypto.ZIL/USD": "zilliqa", + "Crypto.IOTA/USD": "iota", + "Crypto.ONE/USD": "harmony", + "Crypto.WAVES/USD": "waves", + "Crypto.XTZ/USD": "tezos", + "Crypto.DASH/USD": "dash", + "Crypto.ZEC/USD": "zcash", + "Crypto.XMR/USD": "monero", + "Crypto.ANC/USD": "anchor-protocol", + "Crypto.APE/USD": "apecoin", + "Crypto.ATLAS/USD": "star-atlas", + "Crypto.AUST/USD": "anchorust", + "Crypto.BETH/USD": "binance-eth", + "Crypto.BRZ/USD": "brz", + "Crypto.BUSD/USD": "binance-usd", + "Crypto.C98/USD": "coin98", + "Crypto.COPE/USD": "cope", + "Crypto.CUSD/USD": "celo-dollar", + "Crypto.FIDA/USD": "bonfida", + "Crypto.FTT/USD": "ftx-token", + "Crypto.GMT/USD": "stepn", + "Crypto.GOFX/USD": "goosefx", + "Crypto.HXRO/USD": "hxro", + "Crypto.INJ/USD": "injective-protocol", + "Crypto.JET/USD": "jet", + "Crypto.LUNA/USD": "terra-luna-2", + "Crypto.LUNC/USD": "terra-luna", + "Crypto.MER/USD": "mercurial", + "Crypto.MIR/USD": "mirror-protocol", + "Crypto.MNGO/USD": "mango-markets", + "Crypto.MSOL/USD": "msol", + "Crypto.ORCA/USD": "orca", + "Crypto.PAI/USD": "parrot-usd", + "Crypto.PORT/USD": "port-finance", + "Crypto.RAY/USD": "raydium", + "Crypto.SBR/USD": "saber", + "Crypto.SCNSOL/USD": "socean-staked-sol", + "Crypto.SLND/USD": "solend", + "Crypto.SNY/USD": "synthetify-token", + "Crypto.SRM/USD": "serum", + "Crypto.STEP/USD": "step-finance", + "Crypto.STSOL/USD": "lido-staked-sol", + "Crypto.TUSD/USD": "true-usd", + "Crypto.USTC/USD": "terrausd", + "Crypto.VAI/USD": "vai", + "Crypto.XVS/USD": "venus", + "Crypto.ZBC/USD": "zebec-protocol", +} + + +def normalize_symbol(symbol: str) -> str: + """Normalize symbol - only remove suffixes separated by / or -.""" + original = symbol.upper().strip() + # Only remove suffixes if they're separated by / or - + for suffix in ["-USD", "/USD", "-USDT", "/USDT", "-USDC", "/USDC"]: + if original.endswith(suffix): + return original[: -len(suffix)].strip() + return original + + +def is_non_canonical(coin_id: str, coin_name: str) -> bool: + """Check if coin is bridged/peg/wrapped (non-canonical).""" + text = (coin_id + " " + coin_name).lower() + return any( + term in text + for term in ["bridged", "peg", "wrapped", "wormhole", "binance-peg", "mapped-"] + ) + + +def normalize_text(text: str) -> str: + """Normalize text for matching (remove separators, lowercase).""" + return ( + text.lower().replace("-", "").replace("_", "").replace(" ", "").replace("/", "") + ) + + +def match_by_description( + description: str, coins: List[Dict[str, Any]] +) -> Optional[Dict[str, Any]]: + """ + Match coin using Pyth description (most reliable method). + Description format: "COIN_NAME / US DOLLAR" or similar. + """ + if not description: + return None + + # Extract words from description (e.g., "UNISWAP / US DOLLAR" -> ["uniswap"]) + # Get the first significant word (usually the coin name) + desc_parts = description.upper().split("/")[0].strip() # Get part before "/" + desc_words = [ + w.replace("-", "").replace("_", "").lower() + for w in desc_parts.replace("-", " ").split() + if len(w) > 2 and w.lower() not in ["usd", "us", "dollar", "euro", "eur", "and"] + ] + + # Also create combined version for multi-word matches (e.g., "USD COIN" -> "usdcoin") + desc_combined = "".join(desc_words) + + # First pass: find exact matches (prefer canonical) + canonical_matches = [] + non_canonical_matches = [] + + for coin in coins: + coin_id_norm = normalize_text(coin["id"]) + coin_name_norm = normalize_text(coin["name"]) + is_non_can = is_non_canonical(coin["id"], coin["name"]) + + # Check exact word match with coin ID (most reliable) + for word in desc_words: + if word == coin_id_norm: + if is_non_can: + non_canonical_matches.append(coin) + else: + # Return immediately for canonical exact match + return coin + + # Check combined description match + if desc_combined == coin_id_norm: + if is_non_can: + non_canonical_matches.append(coin) + else: + canonical_matches.append(coin) + + # Check if coin name matches + if coin_name_norm in desc_combined or any( + word == coin_name_norm for word in desc_words + ): + if is_non_can: + non_canonical_matches.append(coin) + else: + canonical_matches.append(coin) + + # Return first canonical match, or first non-canonical if no canonical found + if canonical_matches: + return canonical_matches[0] + if non_canonical_matches: + return non_canonical_matches[0] + + return None + + +def score_coin(symbol: str, coin: Dict[str, Any], description: str = "") -> float: + """Score a coin match. Higher is better.""" + coin_id = coin["id"].lower() + coin_name = coin["name"].lower() + symbol_lower = symbol.lower() + + # Heavy penalty for non-canonical coins + if is_non_canonical(coin_id, coin_name): + base = 0.1 + else: + base = 1.0 + + # Penalty for generic names (name == symbol) + if coin_name == symbol_lower: + base *= 0.3 + + # Bonus if description matches + if description: + desc_norm = normalize_text(description) + coin_id_norm = normalize_text(coin_id) + if coin_id_norm in desc_norm: + return base + 0.5 + coin_name_norm = normalize_text(coin_name) + if coin_name_norm in desc_norm: + return base + 0.3 + + # Similarity bonus + similarity = SequenceMatcher(None, symbol_lower, coin_name).ratio() + return base * (0.5 + similarity * 0.5) + + +def find_coingecko_match( + pyth_base: str, + coin_lookup: Dict[str, Any], + description: str = "", +) -> Tuple[Optional[str], float, str]: + """Find the best CoinGecko match for a Pyth base symbol. + + Returns: + Tuple of (coin_id, confidence_score, match_type) + """ + normalized = normalize_symbol(pyth_base) + + # Strategy 1: Exact symbol match + if normalized in coin_lookup["by_symbol"]: + coins = coin_lookup["by_symbol"][normalized] + + if len(coins) == 1: + return coins[0]["id"], 1.0, "exact_symbol" + + # Try description matching first (most reliable) + desc_match = match_by_description(description, coins) + if desc_match: + return desc_match["id"], 1.0, "exact_symbol" + + # Score all coins and pick best + best_coin = None + best_score = -1.0 + + for coin in coins: + score = score_coin(normalized, coin, description) + if score > best_score: + best_score = score + best_coin = coin + + if best_coin: + return best_coin["id"], 1.0, "exact_symbol" + + # Strategy 2: Fuzzy match on symbol and coin ID + best_coin = None + best_score = 0.0 + + for coin in coin_lookup["all_coins"]: + coin_symbol = coin["symbol"].upper() + coin_id = coin["id"].upper() + + # Check exact match with coin ID first (most reliable) + if normalized == coin_id: + return coin["id"], 1.0, "fuzzy_symbol" + + # Check similarity with both symbol and ID, prefer ID matches + symbol_score = SequenceMatcher(None, normalized, coin_symbol).ratio() + id_score = SequenceMatcher(None, normalized, coin_id).ratio() + + # Use the better of the two scores, with slight preference for ID matches + score = max(symbol_score, id_score * 1.01) # 1% bonus for ID matches + + if score > best_score and score >= 0.7: + best_score = score + best_coin = coin + + if best_coin: + # If we found an exact ID match, return 100% confidence + if normalized == best_coin["id"].upper(): + return best_coin["id"], 1.0, "fuzzy_symbol" + return best_coin["id"], best_score, "fuzzy_symbol" + + return None, 0.0, "no_match" + + +def validate_known_mappings( + mapping: Dict[str, str], coin_lookup: Dict[str, Any] +) -> Tuple[bool, List[str]]: + """Validate that known mappings match the generated mapping.""" + errors = [] + + for symbol, expected_id in KNOWN_MAPPINGS.items(): + if symbol in mapping: + actual_id = mapping[symbol] + if actual_id != expected_id: + expected_coin = coin_lookup["by_id"].get(expected_id, {}) + actual_coin = coin_lookup["by_id"].get(actual_id, {}) + expected_name = expected_coin.get("name", expected_id) + actual_name = actual_coin.get("name", actual_id) + + errors.append( + f"❌ VALIDATION FAILED: {symbol} mapped to '{actual_id}' ({actual_name}) " + f"but expected '{expected_id}' ({expected_name})" + ) + + return len(errors) == 0, errors + + +def get_pyth_price_feeds() -> List[Dict[str, Any]]: + """Fetch all price feeds from Pyth Hermes API.""" + logger.info(f"Fetching price feeds from {HERMES_API_URL}...") + try: + response = requests.get(HERMES_API_URL, timeout=30) + response.raise_for_status() + feeds = response.json() + logger.info(f"Fetched {len(feeds)} price feeds from Hermes API") + time.sleep(1) # Rate limit protection + return feeds + except Exception as e: + logger.error(f"Failed to fetch price feeds from Hermes API: {e}") + sys.exit(1) + + +def get_coingecko_coin_list() -> Dict[str, Any]: + """Fetch CoinGecko coin list and create lookup dictionaries.""" + logger.info("Fetching CoinGecko coin list...") + try: + coins = COINGECKO_API.get_coins_list() + logger.info(f"Fetched {len(coins)} coins from CoinGecko") + time.sleep(1) # Rate limit protection + + by_id = {coin["id"]: coin for coin in coins} + by_symbol = {} + for coin in coins: + symbol_upper = coin["symbol"].upper() + if symbol_upper not in by_symbol: + by_symbol[symbol_upper] = [] + by_symbol[symbol_upper].append(coin) + + return {"by_id": by_id, "by_symbol": by_symbol, "all_coins": coins} + except Exception as e: + logger.error(f"Failed to fetch CoinGecko coin list: {e}") + sys.exit(1) + + +def get_hermes_prices(symbol_to_feed_id: Dict[str, str]) -> Dict[str, float]: + """Get latest prices from Hermes API for mapped symbols.""" + logger.info("Fetching prices from Hermes API...") + hermes_prices = {} + + try: + # Get price updates for all feeds in batches + feed_ids = list(symbol_to_feed_id.values()) + batch_size = 50 + + for i in range(0, len(feed_ids), batch_size): + batch = feed_ids[i : i + batch_size] + query_string = "?" + "&".join(f"ids[]={feed_id}" for feed_id in batch) + url = f"{HERMES_API_URL.replace('/price_feeds', '/updates/price/latest')}{query_string}" + + response = requests.get(url, timeout=30) + response.raise_for_status() + data = response.json() + + for feed_data in data.get("parsed", []): + feed_id = feed_data.get("id") + price_info = feed_data.get("price", {}) + if price_info: + price_str = price_info.get("price", "0") + expo = price_info.get("expo", 0) + try: + price = int(price_str) + # Convert to actual price: price * 10^expo + actual_price = price * (10**expo) + if actual_price > 0: + hermes_prices[feed_id] = actual_price + except (ValueError, TypeError): + continue + + time.sleep(1) # Rate limit protection - wait 1s after each batch + + logger.info(f"Fetched {len(hermes_prices)} prices from Hermes API") + except Exception as e: + logger.warning(f"Failed to fetch Hermes prices: {e}") + + return hermes_prices + + +def get_coingecko_prices(mapping: Dict[str, str]) -> Dict[str, float]: + """Get prices from CoinGecko for mapped coins.""" + logger.info("Fetching prices from CoinGecko...") + coingecko_prices = {} + + try: + # Get unique coin IDs + coin_ids = list(set(mapping.values())) + + # CoinGecko API can handle up to ~1000 IDs at once, but let's batch to be safe + batch_size = 200 + for i in range(0, len(coin_ids), batch_size): + batch = coin_ids[i : i + batch_size] + prices = COINGECKO_API.get_price( + ids=batch, vs_currencies="usd", include_last_updated_at=False + ) + + for coin_id, price_data in prices.items(): + if "usd" in price_data: + coingecko_prices[coin_id] = price_data["usd"] + + time.sleep(2) # Rate limit protection - wait 1s after each batch + + logger.info(f"Fetched {len(coingecko_prices)} prices from CoinGecko") + except Exception as e: + logger.warning(f"Failed to fetch CoinGecko prices: {e}") + + return coingecko_prices + + +def validate_prices( + mapping: Dict[str, str], + pyth_feeds: List[Dict[str, Any]], + max_deviation_percent: float = 10.0, +) -> Tuple[List[str], List[str], Dict[str, Dict[str, float]]]: + """ + Validate prices by comparing Hermes and CoinGecko prices. + Returns tuple of (warnings, price_mismatch_symbols, price_details) for significant price differences. + price_details is a dict mapping symbol to {'hermes_price': float, 'coingecko_price': float, 'deviation': float} + """ + warnings = [] + price_mismatch_symbols = [] + price_details = {} + + # Map symbols to feed IDs + symbol_to_feed_id = {} + for feed in pyth_feeds: + attrs = feed.get("attributes", {}) + if attrs.get("asset_type") == "Crypto": + symbol = attrs.get("symbol", "") + if symbol and symbol in mapping: + symbol_to_feed_id[symbol] = feed.get("id") + + if not symbol_to_feed_id: + return warnings, price_mismatch_symbols, price_details + + # Get prices from both sources + hermes_prices = get_hermes_prices(symbol_to_feed_id) + coingecko_prices = get_coingecko_prices(mapping) + + # Compare prices + compared = 0 + mismatches = 0 + for symbol, coin_id in mapping.items(): + feed_id = symbol_to_feed_id.get(symbol) + + if feed_id is None: + continue + + hermes_price = hermes_prices.get(feed_id) + cg_price = coingecko_prices.get(coin_id) + + # Skip if either price is missing + if not hermes_price or not cg_price: + continue + + # Skip if CoinGecko price is 0 (coin might not be actively traded) + if cg_price <= 0: + continue + + compared += 1 + deviation = abs(hermes_price - cg_price) / cg_price * 100 + + # Only warn if deviation is significant and price is meaningful + if deviation > max_deviation_percent and cg_price >= 0.01: + mismatches += 1 + warnings.append( + f"⚠️ {symbol} ({coin_id}): Price mismatch - Hermes: ${hermes_price:,.5f}, " + f"CoinGecko: ${cg_price:,.5f} (deviation: {deviation:.2f}%)" + ) + price_mismatch_symbols.append(symbol) + price_details[symbol] = { + "hermes_price": hermes_price, + "coingecko_price": cg_price, + "deviation": deviation, + } + logger.warning( + f" Price mismatch: {symbol} ({coin_id}) - " + f"Hermes: ${hermes_price:,.5f} | CoinGecko: ${cg_price:,.5f} | " + f"Deviation: {deviation:.2f}%" + ) + + if compared > 0: + logger.info(f"Compared prices for {compared} symbols") + if mismatches > 0: + logger.warning( + f"Found {mismatches} price mismatches (deviation > {max_deviation_percent}%)" + ) + + return warnings, price_mismatch_symbols, price_details + + +def build_mapping( + validate_prices_flag: bool = False, max_deviation: float = 10.0 +) -> Tuple[ + Dict[str, str], Dict[str, float], List[str], List[str], Dict[str, Dict[str, float]] +]: + """Build the CoinGecko mapping from Pyth feeds. + + Returns: + Tuple of (mapping, confidence_scores, warnings, price_mismatch_symbols, price_details) + price_details maps symbol to {'hermes_price': float, 'coingecko_price': float, 'deviation': float} + """ + pyth_feeds = get_pyth_price_feeds() + coin_lookup = get_coingecko_coin_list() + + # Extract Crypto symbols with base and descriptions + crypto_data = {} + for feed in pyth_feeds: + attrs = feed.get("attributes", {}) + if attrs.get("asset_type") == "Crypto": + symbol = attrs.get("symbol", "") + base = attrs.get("base", "") + quote_currency = attrs.get("quote_currency", "") + + if quote_currency != "USD": + continue + + if symbol and base: + if symbol not in crypto_data: + crypto_data[symbol] = { + "base": base, + "description": attrs.get("description", ""), + } + + logger.info(f"Found {len(crypto_data)} unique Crypto symbols quoted in USD") + + # Build mapping + mapping = {} + confidence_scores = {} # Track confidence scores for each symbol + warnings = [] + + for symbol in sorted(crypto_data.keys()): + base = crypto_data[symbol]["base"] + description = crypto_data[symbol]["description"] + api_id, score, match_type = find_coingecko_match(base, coin_lookup, description) + + if api_id: + mapping[symbol] = api_id + confidence_scores[symbol] = score + if score < 1.0: + warnings.append( + f"⚠️ {symbol}: Match confidence {score:.2%} ({match_type}) - " + f"matched to '{api_id}'" + ) + else: + warnings.append(f"❌ {symbol}: No match found in CoinGecko") + + # Validate against known mappings + is_valid, validation_errors = validate_known_mappings(mapping, coin_lookup) + if not is_valid: + logger.error("\n" + "=" * 60) + logger.error( + "VALIDATION FAILED: Known mappings do not match generated mappings!" + ) + logger.error("=" * 60) + for error in validation_errors: + logger.error(error) + logger.error("\nThis indicates the matching algorithm needs improvement.") + logger.error( + "Please fix the matching logic before using the generated mapping." + ) + return mapping, confidence_scores, warnings + validation_errors, [] + + logger.info("✓ Validation passed: All known mappings match generated mappings") + + # Validate prices if requested + price_mismatch_symbols = [] + price_details = {} + if validate_prices_flag: + price_warnings, price_mismatch_symbols, price_details = validate_prices( + mapping, pyth_feeds, max_deviation + ) + warnings.extend(price_warnings) + + return mapping, confidence_scores, warnings, price_mismatch_symbols, price_details + + +def load_existing_mapping(file_path: str) -> Dict[str, str]: + """Load existing mapping file if it exists.""" + try: + with open(file_path, "r") as f: + content = f.read().strip() + if content.startswith("{"): + data = json.loads(content) + # Handle both old format (dict) and new format (string) + if data and isinstance(list(data.values())[0], dict): + # Convert old format to new format + return { + k: v.get("api", v.get("market", "")) for k, v in data.items() + } + return data + except FileNotFoundError: + pass + except (json.JSONDecodeError, (KeyError, IndexError)): + logger.warning(f"Could not parse existing mapping file {file_path}") + return {} + + +def compare_mappings( + new_mapping: Dict[str, str], existing_mapping: Dict[str, str] +) -> List[str]: + """Compare new mapping with existing and return differences.""" + differences = [] + all_symbols = set(list(new_mapping.keys()) + list(existing_mapping.keys())) + + for symbol in all_symbols: + new_entry = new_mapping.get(symbol) + existing_entry = existing_mapping.get(symbol) + + if new_entry and existing_entry: + if new_entry != existing_entry: + differences.append( + f" {symbol}: Changed from '{existing_entry}' to '{new_entry}'" + ) + elif new_entry and not existing_entry: + differences.append(f" {symbol}: New entry -> '{new_entry}'") + elif existing_entry and not new_entry: + differences.append(f" {symbol}: Removed (was '{existing_entry}')") + + return differences + + +def main() -> int: + """Main function.""" + import argparse + + parser = argparse.ArgumentParser( + description="Build CoinGecko mapping file from Pyth Hermes API" + ) + parser.add_argument( + "-o", + "--output", + default="coingecko_mapping.json", + help="Output file path (default: coingecko_mapping.json)", + ) + parser.add_argument( + "-e", "--existing", help="Path to existing mapping file to compare against" + ) + parser.add_argument( + "--no-validate-prices", + action="store_true", + help="Skip price validation (by default, prices are validated)", + ) + parser.add_argument( + "--max-price-deviation", + type=float, + default=1.0, + help="Maximum price deviation percentage to warn about (default: 1.0%%)", + ) + args = parser.parse_args() + + logger.info("Starting CoinGecko mapping generation...") + + existing_mapping = {} + if args.existing: + existing_mapping = load_existing_mapping(args.existing) + if existing_mapping: + logger.info( + f"Loaded {len(existing_mapping)} existing mappings from {args.existing}" + ) + + ( + mapping, + confidence_scores, + warnings, + price_mismatch_symbols, + price_details, + ) = build_mapping( + validate_prices_flag=not args.no_validate_prices, + max_deviation=args.max_price_deviation, + ) + + # Check if validation failed + validation_failed = any("VALIDATION FAILED" in w for w in warnings) + if validation_failed: + logger.error("\nExiting with error code due to validation failures.") + return 1 + + # Filter out symbols with low confidence (< 1.0) or price mismatches + excluded_symbols = set() + excluded_low_confidence = [] + excluded_price_mismatch = [] + + # Find symbols with confidence < 1.0 + for symbol, score in confidence_scores.items(): + if score < 1.0: + excluded_symbols.add(symbol) + excluded_low_confidence.append(symbol) + + # Find symbols with price mismatches + for symbol in price_mismatch_symbols: + excluded_symbols.add(symbol) + excluded_price_mismatch.append(symbol) + + # Create filtered mapping (only high confidence, no price mismatches) + filtered_mapping = { + symbol: coin_id + for symbol, coin_id in mapping.items() + if symbol not in excluded_symbols + } + + # Log excluded entries for manual review + if excluded_low_confidence or excluded_price_mismatch: + logger.warning("\n" + "=" * 60) + logger.warning("EXCLUDED ENTRIES (for manual review):") + logger.warning("=" * 60) + + if excluded_low_confidence: + logger.warning( + f"\n⚠️ Low confidence matches (< 100%) - {len(excluded_low_confidence)} entries:" + ) + for symbol in sorted(excluded_low_confidence): + coin_id = mapping.get(symbol, "N/A") + score = confidence_scores.get(symbol, 0.0) + logger.warning(f" {symbol}: {coin_id} (confidence: {score:.2%})") + + if excluded_price_mismatch: + logger.warning( + f"\n⚠️ Price mismatches - {len(excluded_price_mismatch)} entries:" + ) + for symbol in sorted(excluded_price_mismatch): + coin_id = mapping.get(symbol, "N/A") + if symbol in price_details: + details = price_details[symbol] + hermes_price = details["hermes_price"] + cg_price = details["coingecko_price"] + deviation = details["deviation"] + logger.warning( + f" {symbol} ({coin_id}): " + f"Hermes: ${hermes_price:,.5f} | " + f"CoinGecko: ${cg_price:,.5f} | " + f"Deviation: {deviation:.2f}%" + ) + else: + logger.warning(f" {symbol}: {coin_id}") + + # Output excluded entries as JSON for easy manual addition + excluded_mapping = { + symbol: mapping[symbol] for symbol in excluded_symbols if symbol in mapping + } + if excluded_mapping: + excluded_file = args.output.replace(".json", "_excluded.json") + with open(excluded_file, "w") as f: + json.dump(excluded_mapping, f, indent=2, sort_keys=True) + logger.warning( + f"\n📝 Excluded entries saved to {excluded_file} for manual review" + ) + logger.warning("=" * 60 + "\n") + + # Output results + logger.info(f"\n{'=' * 60}") + logger.info(f"Generated mapping for {len(mapping)} symbols") + if excluded_symbols: + logger.info( + f"Excluded {len(excluded_symbols)} entries (low confidence or price mismatch)" + ) + logger.info(f"Final mapping contains {len(filtered_mapping)} symbols") + logger.info(f"{'=' * 60}\n") + + # Compare with existing if provided + if existing_mapping: + differences = compare_mappings(filtered_mapping, existing_mapping) + if differences: + logger.info(f"Found {len(differences)} differences from existing mapping:") + for diff in differences: + logger.info(diff) + logger.info("") + + # Print warnings (excluding excluded entries from warnings count) + other_warnings = [ + w + for w in warnings + if not any(symbol in w for symbol in excluded_symbols) + and "VALIDATION FAILED" not in w + ] + if other_warnings: + logger.warning(f"Found {len(other_warnings)} other warnings:") + for warning in other_warnings: + logger.warning(warning) + logger.info("") + + # Output JSON (only high confidence, no price mismatches) + with open(args.output, "w") as f: + json.dump(filtered_mapping, f, indent=2, sort_keys=True) + + logger.info(f"✓ Mapping saved to {args.output}") + + # Summary + fuzzy_matches = len(excluded_low_confidence) + no_matches = len([w for w in warnings if "No match found" in w]) + exact_matches = len(filtered_mapping) + + logger.info(f"\nSummary:") + logger.info(f" Total symbols processed: {len(mapping)}") + logger.info(f" Included in final mapping: {exact_matches} (exact matches only)") + logger.info(f" Excluded - low confidence: {fuzzy_matches}") + logger.info(f" Excluded - price mismatch: {len(excluded_price_mismatch)}") + logger.info(f" No matches found: {no_matches}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_checks_price_feed.py b/tests/test_checks_price_feed.py index b774c7a..3fdfb62 100644 --- a/tests/test_checks_price_feed.py +++ b/tests/test_checks_price_feed.py @@ -18,12 +18,6 @@ def test_price_feed_offline_check(): confidence_interval_aggregate=10.0, coingecko_price=1005.0, coingecko_update=0, - crosschain_price={ - "price": 1003.0, - "conf": 10.0, - "publish_time": 123, - "snapshot_time": 123, - }, ) assert PriceFeedOfflineCheck( diff --git a/tests/test_checks_publisher.py b/tests/test_checks_publisher.py index 80dc8d5..fa4657a 100644 --- a/tests/test_checks_publisher.py +++ b/tests/test_checks_publisher.py @@ -46,7 +46,7 @@ def make_publisher_state( def test_publisher_price_check(): def check_is_ok( - state: PublisherState, max_aggregate_distance: int, max_slot_distance: int + state: PublisherState, max_aggregate_distance: float, max_slot_distance: int ) -> bool: return PublisherPriceCheck( state,