From cf2d641631cfc1d8ab162fc136c537bdd537800e Mon Sep 17 00:00:00 2001 From: emidev98 Date: Thu, 15 Feb 2024 17:56:10 +0200 Subject: [PATCH 1/3] feat: refact osmo to pull individual pools --- config/default_config.go | 30 ++- .../parser/internal/coingecko/coingecko.go | 1 + internal/provider/internal/osmosis/osmosis.go | 245 ++++++++---------- internal/types/osmosis_pool.go | 67 +++++ 4 files changed, 191 insertions(+), 152 deletions(-) create mode 100644 internal/types/osmosis_pool.go diff --git a/config/default_config.go b/config/default_config.go index b1f1858..fe5ed26 100644 --- a/config/default_config.go +++ b/config/default_config.go @@ -4,7 +4,7 @@ var DefaultPriceServerConfig = Config{ Port: 8532, MetricsPort: 8533, Sentry: "", - ProviderPriority: []string{"astroport", "binance", "huobi", "coingecko", "kucoin", "bitfinex", "kraken", "okx", "osmosis" /*"bitstamp", */, "bybit" /*"bittrex",*/, "exchangerate", "frankfurter", "fer"}, + ProviderPriority: []string{ /*"astroport", "binance", "huobi", "kucoin", "bitfinex", "kraken", "okx", */ "coingecko", "osmosis", "bitstamp" /* "bybit" "bittrex", "exchangerate", "frankfurter", "fer"*/}, Providers: map[string]ProviderConfig{ "astroport": { Interval: 30, @@ -1190,9 +1190,10 @@ var DefaultPriceServerConfig = Config{ }, }, "coingecko": { - Interval: 6, + Interval: 30, Timeout: 10, Symbols: []string{ + "osmosis", "bitcoin", "ethereum", "binancecoin", @@ -1217,20 +1218,21 @@ var DefaultPriceServerConfig = Config{ }, }, "osmosis": { - Interval: 6, + Interval: 30, Symbols: []string{ - "ATOM/USDC", - "AKT/USDC", - "JUNO/USDC", - "SCRT/USDC", - "STARS/USDC", + "ATOM/OSMO", + "AKT/OSMO", + "JUNO/OSMO", + "SCRT/OSMO", + "STARS/OSMO", + "USDC/OSMO", + "INJ/OSMO", + "LUNA/OSMO", + "KAVA/OSMO", + "LINK/OSMO", + "LUNC/OSMO", + "ASH/USDC", "OSMO/USDC", - "INJ/USDC", - "LUNA/USDC", - "KAVA/USDC", - "LINK/USDC", - "DOT/USDC", - "LUNC/USDC", }, }, "coinbase": { diff --git a/internal/parser/internal/coingecko/coingecko.go b/internal/parser/internal/coingecko/coingecko.go index 68a2faf..af6f1f3 100644 --- a/internal/parser/internal/coingecko/coingecko.go +++ b/internal/parser/internal/coingecko/coingecko.go @@ -28,6 +28,7 @@ var COIN_GECKO_MAPPING = map[string]string{ "white-whale": "WHALE", // White Whale chain "switcheo": "SWTH", // Carbon chain "stride-staked-luna": "STLUNA", // Stride chain + "osmosis": "OSMO", } func ParseSymbol(symbol string) (string, string, error) { diff --git a/internal/provider/internal/osmosis/osmosis.go b/internal/provider/internal/osmosis/osmosis.go index ff71c72..73ac147 100644 --- a/internal/provider/internal/osmosis/osmosis.go +++ b/internal/provider/internal/osmosis/osmosis.go @@ -6,15 +6,14 @@ import ( "io" "log" "net/http" - "strconv" + "strings" "sync" "time" + sdktypes "github.com/cosmos/cosmos-sdk/types" "github.com/terra-money/oracle-feeder-go/config" - "github.com/terra-money/oracle-feeder-go/internal/parser" internal_types "github.com/terra-money/oracle-feeder-go/internal/types" "github.com/terra-money/oracle-feeder-go/pkg/types" - "golang.org/x/exp/maps" ) type OsmosisEndpoint struct { @@ -28,28 +27,33 @@ type OsmosisProvider struct { mu *sync.Mutex } -var endpoints []OsmosisEndpoint +var endpoints = []OsmosisEndpoint{{ + url: "https://osmosis-api.polkachu.com/osmosis/gamm/v1beta1/pools/${POOL_ID}", + used: false, +}, { + url: "https://osmosis-api.polkachu.com/osmosis/gamm/v1beta1/pools/${POOL_ID}", + used: false, +}, { + url: "https://lcd-osmosis.tfl.foundation/osmosis/gamm/v1beta1/pools/${POOL_ID}", + used: false, +}} var whiteListPoolIds = map[string]string{ - "ATOM/USDC": "1", - "AKT/USDC": "3", - // "CRO/USDC": "9", // DOUBLE CHECK - "JUNO/USDC": "497", - // "USTC/USDC": "560", // DOUBLE CHECK - "SCRT/USDC": "584", - "STARS/USDC": "604", - // "DAI/USDC": "674", // DOUBLE CHECK - "OSMO/USDC": "678", - // "EVMOS/USDC": "722", // DOUBLE CHECK - "INJ/USDC": "725", - "LUNA/USDC": "726", - "KAVA/USDC": "730", - "LINK/USDC": "731", - // "MKR/USDC": "733", // DOUBLE CHECK - // "DOT/USDC": "773", // DOUBLE CHECK - "LUNC/USDC": "800", + "ATOM/OSMO": "1", + "AKT/OSMO": "3", + "JUNO/OSMO": "497", + "SCRT/OSMO": "584", + "STARS/OSMO": "604", + "USDC/OSMO": "678", + "INJ/OSMO": "725", + "LUNA/OSMO": "726", + "KAVA/OSMO": "730", + "LINK/OSMO": "731", + "LUNC/OSMO": "800", + "ASH/USDC": "1360", + "OSMO/USDC": "1464", } -var idToSymbols = make(map[string]string) +var idsBySymbol = make(map[string]string) func NewOsmosisProvider(config *config.ProviderConfig, stopCh <-chan struct{}) (*OsmosisProvider, error) { mu := sync.Mutex{} @@ -59,16 +63,8 @@ func NewOsmosisProvider(config *config.ProviderConfig, stopCh <-chan struct{}) ( mu: &mu, } - endpoints = append(endpoints, OsmosisEndpoint{ - "https://osmosis-api.polkachu.com/osmosis/gamm/v1beta1/pools?pagination.limit=801", - false, - }) - endpoints = append(endpoints, OsmosisEndpoint{ - "https://lcd.osmosis.zone/osmosis/gamm/v1beta1/pools?pagination.limit=801", - false, - }) for k, v := range whiteListPoolIds { - idToSymbols[v] = k + idsBySymbol[v] = k } go func() { @@ -88,23 +84,6 @@ func NewOsmosisProvider(config *config.ProviderConfig, stopCh <-chan struct{}) ( return provider, nil } -func rotateUrl() (string, error) { - if len(endpoints) == 0 { - return "", fmt.Errorf("No endpoints") - } - for i := range endpoints { - if !endpoints[i].used { - endpoints[i].used = true - return endpoints[i].url, nil - } - } - for i := range endpoints { - endpoints[i].used = false - } - endpoints[0].used = true - return endpoints[0].url, nil -} - func (p *OsmosisProvider) GetPrices() map[string]types.PriceByPair { result := make(map[string]types.PriceByPair) p.mu.Lock() @@ -122,115 +101,105 @@ func (p *OsmosisProvider) GetPrices() map[string]types.PriceByPair { } func (p *OsmosisProvider) fetchAndParse() { - msg, err := fetchPrices(p.config.Symbols) - if err != nil { - log.Printf("%v", err) - } else { - prices, err := parseJSON(msg) + for id, symbol := range idsBySymbol { + res, err := fetchPrice(id) if err != nil { log.Printf("%v", err) - } else { - p.mu.Lock() - maps.Copy(p.priceBySymbol, prices) - p.mu.Unlock() + continue + } + var generic internal_types.GenericPoolResponse + if err := json.Unmarshal(res, &generic); err != nil { + fmt.Println("Error:", err) + continue + } + price, err := p.parsePrice(generic, res) + if err != nil { + continue } - } -} - -func fetchPrices(symbols []string) ([]interface{}, error) { - url, err := rotateUrl() - if err != nil { - return nil, err - } - client := &http.Client{Timeout: time.Second * 15} - resp, err := client.Get(url) - if err != nil { - return nil, err - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - jsonObj := make(map[string]interface{}) - err = json.Unmarshal(body, &jsonObj) - if err != nil { - log.Printf("parse response error: %v\n", string(body)) - return nil, err - } - data, ok := jsonObj["pools"].([]interface{}) - if !ok { - log.Printf("no pools: %v\n", string(body)) - return nil, err + p.mu.Lock() + p.priceBySymbol[symbol] = internal_types.PriceBySymbol{ + Symbol: symbol, + Price: price, + Base: strings.Split(symbol, "/")[0], + Quote: strings.Split(symbol, "/")[1], + Timestamp: uint64(time.Now().Unix()), + } + p.mu.Unlock() } - return data, nil } -func parseJSON(msg []interface{}) (map[string]internal_types.PriceBySymbol, error) { - prices := make(map[string]internal_types.PriceBySymbol) - now := time.Now() - - osmoPrice := 0.0 - osmoPair := "OSMO/USDC" +func (*OsmosisProvider) parsePrice(generic internal_types.GenericPoolResponse, res []byte) (float64, error) { + switch generic.Pool.Type { + case "/osmosis.concentratedliquidity.v1beta1.Pool": + var pool internal_types.OsmosisPoolResponse + if err := json.Unmarshal(res, &pool); err != nil { + return 0, err + } - for _, value := range msg { - item := value.(map[string]interface{}) - poolId := item["id"].(string) - symbol, ok := idToSymbols[poolId] - if !ok { - continue + // get the first 18 positons of the price to avoid overflow + price := pool.Pool.CurrentSqrtPrice + if len(price) >= 18 { + price = pool.Pool.CurrentSqrtPrice[:18] } - base, quote, err := parser.ParseSymbol("osmosis", symbol) + parsedPrice, err := sdktypes.NewDecFromStr(price) if err != nil { - log.Printf("%v", err) - continue + return 0, err } - assets, ok := item["pool_assets"].([]interface{}) - if !ok || len(assets) != 2 { - log.Printf("invalid pool_assets: %v\n", item) - continue + return parsedPrice.Power(2).Float64() + case "/osmosis.gamm.v1beta1.Pool": + var pool internal_types.OsmosisGammPoolResponse + if err := json.Unmarshal(res, &pool); err != nil { + return 0, err } - first := assets[0].(map[string]interface{})["token"].(map[string]interface{}) - firstAmount, err := strconv.ParseUint(first["amount"].(string), 10, 64) - if err != nil { - continue + + // get the first 18 positons of the price to avoid overflow + firstTokenPrice := pool.Pool.PoolAssets[0].Token.Amount + if len(firstTokenPrice) >= 18 { + firstTokenPrice = firstTokenPrice[:18] } - second := assets[1].(map[string]interface{})["token"].(map[string]interface{}) - // secondDenom := second["denom"].(string) - secondAmount, err := strconv.ParseUint(second["amount"].(string), 10, 64) + parsedFirstTokenPrice, err := sdktypes.NewDecFromStr(firstTokenPrice) if err != nil { - continue - } - price := 0.0 - // log.Printf("secondDenom: %s base: %s %v quote: %s %v\n", secondDenom, base, firstAmount, quote, secondAmount) - if firstAmount > 0 && secondAmount > 0 { - // if secondDenom == "uosmo" { - if symbol == osmoPair { - price = float64(firstAmount) / float64(secondAmount) - } else { - price = float64(secondAmount) / float64(firstAmount) - } + return 0, err } - if symbol == osmoPair { - osmoPrice = price + secondTokenPrice := pool.Pool.PoolAssets[1].Token.Amount + if len(secondTokenPrice) >= 18 { + secondTokenPrice = secondTokenPrice[:18] } - prices[symbol] = internal_types.PriceBySymbol{ - Exchange: "osmosis", - Symbol: symbol, - Base: base, - Quote: quote, - Price: price, - Timestamp: uint64(now.UnixMilli()), + parsedSecondTokenPrice, err := sdktypes.NewDecFromStr(secondTokenPrice) + if err != nil { + return 0, err } + return parsedSecondTokenPrice.Quo(parsedFirstTokenPrice).Float64() + default: + return 0, fmt.Errorf("unknown pool type: %s", generic.Pool.Type) + } +} + +func fetchPrice(poolId string) (res []byte, err error) { + url, err := rotateUrl() + if err != nil { + return nil, err } - if osmoPrice == 0 { - return nil, fmt.Errorf("no osmo price") + client := &http.Client{Timeout: time.Second * 15} + resp, err := client.Get(strings.Replace(url, "${POOL_ID}", poolId, 1)) + if err != nil { + return nil, err } - for _, pairPrice := range prices { - if pairPrice.Symbol != osmoPair { - pairPrice.Price = pairPrice.Price * osmoPrice - prices[pairPrice.Symbol] = pairPrice + + return io.ReadAll(resp.Body) +} + +func rotateUrl() (string, error) { + for i := range endpoints { + if !endpoints[i].used { + endpoints[i].used = true + return endpoints[i].url, nil } } - return prices, nil + for i := range endpoints { + endpoints[i].used = false + } + endpoints[0].used = true + return endpoints[0].url, nil } diff --git a/internal/types/osmosis_pool.go b/internal/types/osmosis_pool.go new file mode 100644 index 0000000..3e26b42 --- /dev/null +++ b/internal/types/osmosis_pool.go @@ -0,0 +1,67 @@ +package types + +// PriceBySymbol represents the price of a trading symbol at a timestamp. +type OsmosisConcentratedLiquidityPool struct { + Type string `json:"@type"` + Address string `json:"address"` + IncentivesAddress string `json:"incentives_address"` + SpreadRewardsAddress string `json:"spread_rewards_address"` + ID string `json:"id"` + CurrentTickLiquidity string `json:"current_tick_liquidity"` + Token0 string `json:"token0"` + Token1 string `json:"token1"` + CurrentSqrtPrice string `json:"current_sqrt_price"` + CurrentTick string `json:"current_tick"` + TickSpacing string `json:"tick_spacing"` + ExponentAtPriceOne string `json:"exponent_at_price_one"` + SpreadFactor string `json:"spread_factor"` + LastLiquidityUpdate string `json:"last_liquidity_update"` +} + +type OsmosisPoolResponse struct { + Pool OsmosisConcentratedLiquidityPool `json:"pool"` +} + +type OsmoToken struct { + Denom string `json:"denom"` + Amount string `json:"amount"` +} + +type PoolAsset struct { + Token OsmoToken `json:"token"` + Weight string `json:"weight"` +} + +type PoolParams struct { + SwapFee string `json:"swap_fee"` + ExitFee string `json:"exit_fee"` + SmoothWeightChangeParams interface{} `json:"smooth_weight_change_params"` +} + +type TotalShares struct { + Denom string `json:"denom"` + Amount string `json:"amount"` +} + +type OsmosisGammPool struct { + Type string `json:"@type"` + Address string `json:"address"` + ID string `json:"id"` + PoolParams PoolParams `json:"pool_params"` + FuturePoolGovernor string `json:"future_pool_governor"` + TotalShares TotalShares `json:"total_shares"` + PoolAssets []PoolAsset `json:"pool_assets"` + TotalWeight string `json:"total_weight"` +} + +type OsmosisGammPoolResponse struct { + Pool OsmosisGammPool `json:"pool"` +} + +type GenericPoolResponse struct { + Pool GenericType `json:"pool"` +} + +type GenericType struct { + Type string `json:"@type"` +} From 050417e66ea0f5e60fcd924cee534c12f92b10d9 Mon Sep 17 00:00:00 2001 From: emidev98 Date: Fri, 16 Feb 2024 08:11:38 +0200 Subject: [PATCH 2/3] feat: enable all APIs --- config/default_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default_config.go b/config/default_config.go index fe5ed26..d34ad8d 100644 --- a/config/default_config.go +++ b/config/default_config.go @@ -4,7 +4,7 @@ var DefaultPriceServerConfig = Config{ Port: 8532, MetricsPort: 8533, Sentry: "", - ProviderPriority: []string{ /*"astroport", "binance", "huobi", "kucoin", "bitfinex", "kraken", "okx", */ "coingecko", "osmosis", "bitstamp" /* "bybit" "bittrex", "exchangerate", "frankfurter", "fer"*/}, + ProviderPriority: []string{"astroport", "binance", "huobi", "kucoin", "bitfinex", "kraken", "okx", "coingecko", "osmosis", "bitstamp", "bybit" /*"bittrex",*/, "exchangerate", "frankfurter", "fer"}, Providers: map[string]ProviderConfig{ "astroport": { Interval: 30, From ffa295d07b822b1e138df99f73c1ff444c9600b0 Mon Sep 17 00:00:00 2001 From: emidev98 Date: Mon, 19 Feb 2024 08:22:33 +0200 Subject: [PATCH 3/3] feat: add AMPWHALE to coingecko --- config/default_config.go | 1 + .../parser/internal/coingecko/coingecko.go | 45 ++++++++++--------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/config/default_config.go b/config/default_config.go index d34ad8d..f5c2dfd 100644 --- a/config/default_config.go +++ b/config/default_config.go @@ -1213,6 +1213,7 @@ var DefaultPriceServerConfig = Config{ "stargaze", "akash-network", "white-whale", + "eris-amplified-whale", "switcheo", "stride-staked-luna", }, diff --git a/internal/parser/internal/coingecko/coingecko.go b/internal/parser/internal/coingecko/coingecko.go index af6f1f3..2aba2a2 100644 --- a/internal/parser/internal/coingecko/coingecko.go +++ b/internal/parser/internal/coingecko/coingecko.go @@ -7,28 +7,29 @@ import ( // symbol to base coin mapping var COIN_GECKO_MAPPING = map[string]string{ - "bitcoin": "BTC", - "ethereum": "ETH", - "binancecoin": "BNB", - "tether": "USDT", - "usd-coin": "USDC", - "binance-usd": "BUSD", - "dai": "DAI", - "okb": "OKB", - "solana": "SOL", - "cosmos": "ATOM", - "terra-luna-2": "LUNA", - "terra-luna": "LUNC", - "terrausd": "USTC", - "injective-protocol": "INJ", - "secret": "SCRT", - "juno-network": "JUNO", - "stargaze": "STARS", - "akash-network": "AKT", - "white-whale": "WHALE", // White Whale chain - "switcheo": "SWTH", // Carbon chain - "stride-staked-luna": "STLUNA", // Stride chain - "osmosis": "OSMO", + "bitcoin": "BTC", + "ethereum": "ETH", + "binancecoin": "BNB", + "tether": "USDT", + "usd-coin": "USDC", + "binance-usd": "BUSD", + "dai": "DAI", + "okb": "OKB", + "solana": "SOL", + "cosmos": "ATOM", + "terra-luna-2": "LUNA", + "terra-luna": "LUNC", + "terrausd": "USTC", + "injective-protocol": "INJ", + "eris-amplified-whale": "AMPWHALE", + "secret": "SCRT", + "juno-network": "JUNO", + "stargaze": "STARS", + "akash-network": "AKT", + "white-whale": "WHALE", // White Whale chain + "switcheo": "SWTH", // Carbon chain + "stride-staked-luna": "STLUNA", // Stride chain + "osmosis": "OSMO", } func ParseSymbol(symbol string) (string, string, error) {