From 1b9c8e2e3a7f9f6608dc0e6d9d7825146af4e357 Mon Sep 17 00:00:00 2001 From: koteld Date: Fri, 19 Jan 2024 19:10:17 +0100 Subject: [PATCH] addressing DeepSource issues and code refactoring and update logging --- .../helpers/horizonhelper/stellarAssetInfo.go | 5 ++++- .../scraper/exchange-scrapers/APIScraper.go | 8 ++----- .../exchange-scrapers/StellarScraper.go | 16 ++++++-------- .../liquidity-scrapers/StellarScraper.go | 10 ++++++--- .../service/assetservice/source/stellar.go | 22 +++++-------------- pkg/utils/getenv.go | 4 ++-- 6 files changed, 27 insertions(+), 38 deletions(-) diff --git a/pkg/dia/helpers/horizonhelper/stellarAssetInfo.go b/pkg/dia/helpers/horizonhelper/stellarAssetInfo.go index 4fcd0443b..cd1190234 100644 --- a/pkg/dia/helpers/horizonhelper/stellarAssetInfo.go +++ b/pkg/dia/helpers/horizonhelper/stellarAssetInfo.go @@ -62,7 +62,10 @@ func GetStellarAssetInfo(client *horizonclient.Client, assetCode, assetIssuer, b log = log.WithField("tomlURL", tomlURL) tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + MaxVersion: 0, + }, } tomlClient := &http.Client{ Transport: tr, diff --git a/pkg/dia/scraper/exchange-scrapers/APIScraper.go b/pkg/dia/scraper/exchange-scrapers/APIScraper.go index 43c3bb62b..dbc78c666 100644 --- a/pkg/dia/scraper/exchange-scrapers/APIScraper.go +++ b/pkg/dia/scraper/exchange-scrapers/APIScraper.go @@ -261,18 +261,14 @@ func NewAPIScraper(exchange string, scrape bool, key string, secret string, relD return NewVelodromeScraper(Exchanges[dia.VelodromeExchange], scrape, relDB) case dia.MaverickExchange: return NewMaverickScraper(Exchanges[dia.MaverickExchange], scrape, relDB) - // case dia.FinageForex: - // return NewFinageForexScraper(Exchanges[dia.FinageForex], scrape, relDB, key, secret) - + //case dia.FinageForex: + // return NewFinageForexScraper(Exchanges[dia.FinageForex], scrape, relDB, key, secret) case dia.MultiChain: return NewBridgeSwapScraper(Exchanges[dia.MultiChain], scrape, relDB) - case "Influx": return NewInfluxScraper(scrape) - case "UniswapHistory": return NewUniswapHistoryScraper(Exchanges[dia.UniswapExchange], scrape, relDB) - case dia.StellarExchange: return NewStellarScraper(Exchanges[dia.StellarExchange], scrape, relDB) default: diff --git a/pkg/dia/scraper/exchange-scrapers/StellarScraper.go b/pkg/dia/scraper/exchange-scrapers/StellarScraper.go index 67f360f19..c9bf548db 100644 --- a/pkg/dia/scraper/exchange-scrapers/StellarScraper.go +++ b/pkg/dia/scraper/exchange-scrapers/StellarScraper.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "strconv" "strings" @@ -81,8 +80,8 @@ func NewStellarScraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB) shutdownDone = make(chan nothing) pairScrapers = make(map[string]*StellarPairScraper) chanTrades = make(chan *dia.Trade) - logger = logrus.WithFields(logrus.Fields{ - "context": "StellarScraper", + logger = logrus.New().WithFields(logrus.Fields{ + "context": "StellarExchangeScraper", "cursor": cursor, }) ) @@ -126,7 +125,7 @@ func (s *StellarScraper) mainLoop() { func (s *StellarScraper) tradeHandler(stellarTrade hProtocol.Trade) { log := s.logger.WithFields(logrus.Fields{ - "context": "StellarScraper.tradeHandler", + "context": "StellarExchangeScraper.tradeHandler", "tradeID": stellarTrade.ID, }) @@ -175,15 +174,14 @@ func (s *StellarScraper) tradeHandler(stellarTrade hProtocol.Trade) { QuoteToken: quoteToken, } - s.logger.Infof("StellarScraper.tradeHandler.stellarTrade %# v", pretty.Formatter(stellarTrade)) - s.logger.Infof("StellarScraper.tradeHandler.diaTrade %# v", pretty.Formatter(diaTrade)) + log.Infof("stellarTrade %# v", pretty.Formatter(stellarTrade)) + log.Infof("diaTrade %# v", pretty.Formatter(diaTrade)) s.chanTrades <- diaTrade } func (s *StellarScraper) getDIAAsset(assetType, assetCode, assetIssuer string) (asset dia.Asset, err error) { s.logger. - WithError(err). WithFields(logrus.Fields{ "assetType": assetType, "assetCode": assetCode, @@ -231,7 +229,7 @@ func (s *StellarScraper) ScrapePair(pair dia.ExchangePair) (PairScraper, error) return nil, s.error } if s.closed { - return nil, errors.New("StellarScraper: Call ScrapePair on closed scraper") + return nil, errors.New("StellarExchangeScraper: Call ScrapePair on closed scraper") } ps := &StellarPairScraper{ parent: s, @@ -270,7 +268,7 @@ func (s *StellarScraper) FetchAvailablePairs() ([]dia.ExchangePair, error) { } }(response.Body) - body, err := ioutil.ReadAll(response.Body) + body, err := io.ReadAll(response.Body) if err != nil { s.logger.WithError(err).Error("failed to read symbols") diff --git a/pkg/dia/scraper/liquidity-scrapers/StellarScraper.go b/pkg/dia/scraper/liquidity-scrapers/StellarScraper.go index 57e1bcdc7..1fa510fc6 100644 --- a/pkg/dia/scraper/liquidity-scrapers/StellarScraper.go +++ b/pkg/dia/scraper/liquidity-scrapers/StellarScraper.go @@ -41,8 +41,8 @@ func NewStellarScraper(exchange dia.Exchange, relDB *models.RelDB, datastore *mo poolChannel = make(chan dia.Pool) doneChannel = make(chan bool) scraper *StellarScraper - logger = log.WithFields(logrus.Fields{ - "context": "StellarScraper", + logger = logrus.New().WithFields(logrus.Fields{ + "context": "StellarLiquidityScraper", "cursor": cursor, }) ) @@ -112,6 +112,11 @@ func (scraper *StellarScraper) fetchPools() { } func (scraper *StellarScraper) getDIAPool(stellarPool hProtocol.LiquidityPool) (dia.Pool, error) { + scraper.logger. + WithFields(logrus.Fields{ + "poolId": stellarPool.ID, + }).Info("fetching pool info") + assetvolumes := make([]dia.AssetVolume, len(stellarPool.Reserves)) for i, stellarAsset := range stellarPool.Reserves { asset, err := scraper.getDIAAsset(stellarAsset.Asset) @@ -142,7 +147,6 @@ func (scraper *StellarScraper) getDIAPool(stellarPool hProtocol.LiquidityPool) ( func (scraper *StellarScraper) getDIAAsset(stellarAsset string) (asset dia.Asset, err error) { scraper.logger. - WithError(err). WithFields(logrus.Fields{ "asset": stellarAsset, }).Info("fetching asset info") diff --git a/pkg/dia/service/assetservice/source/stellar.go b/pkg/dia/service/assetservice/source/stellar.go index 1518e8097..a62773782 100644 --- a/pkg/dia/service/assetservice/source/stellar.go +++ b/pkg/dia/service/assetservice/source/stellar.go @@ -5,7 +5,6 @@ import ( "github.com/diadata-org/diadata/pkg/dia" "github.com/diadata-org/diadata/pkg/dia/helpers/horizonhelper" - models "github.com/diadata-org/diadata/pkg/model" "github.com/diadata-org/diadata/pkg/utils" "github.com/stellar/go/clients/horizonclient" ) @@ -20,37 +19,26 @@ type StellarAssetSource struct { assetChannel chan dia.Asset doneChannel chan bool blockchain string - relDB *models.RelDB cursor *string limit uint } func NewStellarAssetSource(exchange dia.Exchange) *StellarAssetSource { - var ( - horizonClient *horizonclient.Client - assetChannel = make(chan dia.Asset) - doneChannel = make(chan bool) - ) - cursor := utils.Getenv(strings.ToUpper(exchange.Name)+"_ASSETS_CURSOR", defaultAssetRequestCursor) limit := utils.GetenvUint(strings.ToUpper(exchange.Name)+"_ASSETS_LIMIT", defaultAssetRequestLimit) - switch exchange.Name { - case dia.StellarExchange: - horizonClient = horizonclient.DefaultPublicNetClient - } + var ( + assetChannel = make(chan dia.Asset) + doneChannel = make(chan bool) + ) - relDB, err := models.NewRelDataStore() - if err != nil { - log.Fatal("make new relational datastore: ", err) - } + horizonClient := horizonclient.DefaultPublicNetClient scraper := &StellarAssetSource{ horizonClient: horizonClient, assetChannel: assetChannel, doneChannel: doneChannel, blockchain: exchange.BlockChain.Name, - relDB: relDB, cursor: &cursor, limit: limit, } diff --git a/pkg/utils/getenv.go b/pkg/utils/getenv.go index 094307887..eb8fea766 100644 --- a/pkg/utils/getenv.go +++ b/pkg/utils/getenv.go @@ -7,7 +7,7 @@ import ( func Getenv(key, fallback string) string { value := os.Getenv(key) - if len(value) == 0 { + if value == "" { return fallback } return value @@ -15,7 +15,7 @@ func Getenv(key, fallback string) string { func GetenvUint(key string, fallback uint) uint { value := os.Getenv(key) - if len(value) == 0 { + if value == "" { return fallback }