add aerodrome v1 and slipstream scraper suites.

cmd/assetCollectionService/go.mod

@@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/assetCollectionService
 go 1.20
 
 require (
-	github.com/diadata-org/diadata v1.4.533
+	github.com/diadata-org/diadata v1.4.539
 	github.com/sirupsen/logrus v1.9.3
 )
 

cmd/assetCollectionService/main.go

@@ -149,7 +149,7 @@ func NewAssetScraper(exchange string, secret string, relDB *models.RelDB) source
 	case dia.OsmosisExchange:
 		return source.NewOsmosisScraper(exchanges[dia.OsmosisExchange])
 	case dia.VelodromeExchange:
-		return source.NewVelodromeAssetSource(exchanges[dia.VelodromeExchange])
+		return source.NewVelodromeAssetSource(exchanges[dia.VelodromeExchange], relDB)
 	case dia.TraderJoeExchangeV2_1:
 		return source.NewTraderJoeAssetSource(exchanges[dia.TraderJoeExchangeV2_1], relDB)
 	case dia.TraderJoeExchangeV2_1Arbitrum:
@@ -170,6 +170,10 @@ func NewAssetScraper(exchange string, secret string, relDB *models.RelDB) source
 		return source.NewJSONReader(exchange, secret)
 	case dia.AyinExchange:
 		return source.NewAyinAssetSource(exchanges[dia.AyinExchange], relDB)
+	case dia.AerodromeSlipstreamExchange:
+		return source.NewVelodromeAssetSource(exchanges[dia.AerodromeSlipstreamExchange], relDB)
+	case dia.AerodromeV1Exchange:
+		return source.NewVelodromeAssetSource(exchanges[dia.AerodromeV1Exchange], relDB)
 	default:
 		return nil
 	}

cmd/exchange-scrapers/collector/go.mod

@@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/exchange-scrapers/collector
 go 1.19
 
 require (
-	github.com/diadata-org/diadata v1.4.529
+	github.com/diadata-org/diadata v1.4.539
 	github.com/segmentio/kafka-go v0.4.35
 	github.com/sirupsen/logrus v1.9.3
 )

cmd/liquidityScraper/go.mod

@@ -3,7 +3,7 @@ module github.com/diadata-org/diadata/liquidityScraper
 go 1.20
 
 require (
-	github.com/diadata-org/diadata v1.4.535
+	github.com/diadata-org/diadata v1.4.539
 	github.com/sirupsen/logrus v1.9.3
 )
 

pkg/dia/Config.go

@@ -122,6 +122,8 @@ const (
 	ThenaV3Exchange                    = "ThenaV3"
 	AyinExchange                       = "Ayin"
 	BitgetExchange                     = "Bitget"
+	AerodromeSlipstreamExchange        = "AerodromeSlipstream"
+	AerodromeV1Exchange                = "AerodromeV1"
 	// FinageForex        = "FinageForex"
 )
 

pkg/dia/scraper/exchange-scrapers/APIScraper.go

@@ -273,6 +273,8 @@ func NewAPIScraper(exchange string, scrape bool, key string, secret string, relD
 		return NewZenlinkScraper(Exchanges[dia.ZenlinkswapExchangeBifrostPolkadot], scrape)
 	case dia.VelodromeExchange:
 		return NewVelodromeScraper(Exchanges[dia.VelodromeExchange], scrape, relDB)
+	case dia.AerodromeV1Exchange:
+		return NewVelodromeScraper(Exchanges[dia.AerodromeV1Exchange], scrape, relDB)
 	case dia.MaverickExchange:
 		return NewMaverickScraper(Exchanges[dia.MaverickExchange], scrape, relDB)
 	case dia.PearlfiExchangeTestnet:
@@ -287,6 +289,10 @@ func NewAPIScraper(exchange string, scrape bool, key string, secret string, relD
 	// 	return NewFinageForexScraper(Exchanges[dia.FinageForex], scrape, relDB, key, secret)
 	case dia.BitgetExchange:
 		return NewBitgetScraper(Exchanges[dia.BitgetExchange], scrape, relDB)
+	case dia.AerodromeSlipstreamExchange:
+		return NewUniswapV3Scraper(Exchanges[dia.AerodromeSlipstreamExchange], scrape, relDB)
+		// case dia.FinageForex:
+		// 	return NewFinageForexScraper(Exchanges[dia.FinageForex], scrape, relDB, key, secret)
 
 	case dia.MultiChain:
 		return NewBridgeSwapScraper(Exchanges[dia.MultiChain], scrape, relDB)

pkg/dia/scraper/exchange-scrapers/UniswapV3Scraper.go

@@ -103,6 +103,8 @@ func NewUniswapV3Scraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB
 		s = makeUniswapV3Scraper(exchange, listenByAddress, "", "", "200", uint64(90593047))
 	case dia.NileV2Exchange:
 		s = makeUniswapV3Scraper(exchange, listenByAddress, "", "", "2000", uint64(1768866))
+	case dia.AerodromeSlipstreamExchange:
+		s = makeUniswapV3Scraper(exchange, listenByAddress, "", "", "2000", uint64(0))
 	}
 
 	s.relDB = relDB

pkg/dia/scraper/exchange-scrapers/VelodromeScraper.go

@@ -21,6 +21,8 @@ import (
 const (
 	velodromeRestDial = ""
 	velodromeWsDial   = ""
+	baseRestDial      = ""
+	baseWsDial        = ""
 )
 
 type VelodromeSwap struct {
@@ -57,6 +59,8 @@ func NewVelodromeScraper(exchange dia.Exchange, scrape bool, relDB *models.RelDB
 	switch exchange.Name {
 	case dia.VelodromeExchange:
 		s = makeVelodromeScraper(exchange, velodromeRestDial, velodromeWsDial, relDB)
+	case dia.AerodromeV1Exchange:
+		s = makeVelodromeScraper(exchange, baseRestDial, baseWsDial, relDB)
 	}
 
 	// Only include pools with (minimum) liquidity bigger than given env var.

pkg/dia/scraper/liquidity-scrapers/ScraperInterface.go

@@ -154,6 +154,10 @@ func NewLiquidityScraper(source string, relDB *models.RelDB, datastore *models.D
 		return NewPlatypusScraper(exchanges[dia.PlatypusExchange], datastore)
 	case dia.VelodromeExchange:
 		return NewVelodromePoolScraper(exchanges[dia.VelodromeExchange], relDB, datastore)
+	case dia.AerodromeV1Exchange:
+		return NewVelodromePoolScraper(exchanges[dia.AerodromeV1Exchange], relDB, datastore)
+	case dia.AerodromeSlipstreamExchange:
+		return NewVelodromePoolScraper(exchanges[dia.AerodromeSlipstreamExchange], relDB, datastore)
 	case dia.MaverickExchange:
 		return NewMaverickScraper(exchanges[dia.MaverickExchange], relDB, datastore)
 	case dia.PearlfiExchangeTestnet:

pkg/dia/scraper/liquidity-scrapers/Velodrome.go

@@ -31,14 +31,18 @@ type VelodromePoolScraper struct {
 
 var (
 	velodromeWaitMilliseconds = "500"
-	velodromeRestDial         = ""
+	restDialOptimism          = ""
 )
 
 func NewVelodromePoolScraper(exchange dia.Exchange, relDB *models.RelDB, datastore *models.DB) (us *VelodromePoolScraper) {
 
 	switch exchange.Name {
 	case dia.VelodromeExchange:
-		us = makeVelodromePoolScraper(exchange, relDB, datastore, velodromeRestDial, velodromeWaitMilliseconds)
+		us = makeVelodromePoolScraper(exchange, relDB, datastore, restDialOptimism, velodromeWaitMilliseconds)
+	case dia.AerodromeV1Exchange:
+		us = makeVelodromePoolScraper(exchange, relDB, datastore, restDialBase, velodromeWaitMilliseconds)
+	case dia.AerodromeSlipstreamExchange:
+		us = makeVelodromePoolScraper(exchange, relDB, datastore, restDialBase, velodromeWaitMilliseconds)
 	}
 
 	go func() {
@@ -111,19 +115,20 @@ func (us *VelodromePoolScraper) GetPoolByID(num int64) (dia.Pool, error) {
 
 	contract, err := velodrome.NewPoolFactoryCaller(common.HexToAddress(us.exchange.Contract), us.RestClient)
 	if err != nil {
-		log.Error(err)
+		log.Error("NewPoolFactoryCaller: ", err)
 		return dia.Pool{}, err
 	}
 
 	pairAddress, err := contract.AllPools(&bind.CallOpts{}, big.NewInt(num))
 	if err != nil {
-		log.Error(err)
+		log.Error("AllPools: ", err)
 		return dia.Pool{}, err
 	}
+	log.Info("pool: ", pairAddress.Hex())
 
 	pool, err := us.GetPoolByAddress(pairAddress)
 	if err != nil {
-		log.Error(err)
+		log.Error("GetPoolByAddress: ", err)
 		return dia.Pool{}, err
 	}
 
@@ -165,7 +170,20 @@ func (us *VelodromePoolScraper) GetPoolByAddress(pairAddress common.Address) (po
 		}
 	}
 
-	// Getting liquidity
+	if us.exchange.Name == dia.AerodromeSlipstreamExchange {
+		us.GetLiquidityUniV3Type(token0, token1, pairAddress, &pool)
+	} else {
+		us.GetLiquidityUniV2Type(token0, token1, pairContract, &pool)
+	}
+
+	pool.Address = pairAddress.Hex()
+	pool.Blockchain = dia.BlockChain{Name: us.blockchain}
+	pool.Exchange = dia.Exchange{Name: us.exchange.Name}
+
+	return pool, nil
+}
+
+func (us *VelodromePoolScraper) GetLiquidityUniV2Type(token0 dia.Asset, token1 dia.Asset, pairContract *velodrome.IPoolCaller, pool *dia.Pool) {
 	liquidity, err := pairContract.GetReserves(&bind.CallOpts{})
 	if err != nil {
 		log.Error("get reserves: ", err)
@@ -191,14 +209,30 @@ func (us *VelodromePoolScraper) GetPoolByAddress(pairAddress common.Address) (po
 
 	// Determine USD liquidity
 	if pool.SufficientNativeBalance(GLOBAL_NATIVE_LIQUIDITY_THRESHOLD) {
-		us.datastore.GetPoolLiquiditiesUSD(&pool, priceCache)
+		us.datastore.GetPoolLiquiditiesUSD(pool, priceCache)
 	}
+}
 
-	pool.Address = pairAddress.Hex()
-	pool.Blockchain = dia.BlockChain{Name: us.blockchain}
-	pool.Exchange = dia.Exchange{Name: us.exchange.Name}
+func (us *VelodromePoolScraper) GetLiquidityUniV3Type(token0 dia.Asset, token1 dia.Asset, pairAddress common.Address, pool *dia.Pool) {
+	balance0Big, err := ethhelper.GetBalanceOf(common.HexToAddress(token0.Address), pairAddress, us.RestClient)
+	if err != nil {
+		log.Error("GetBalanceOf: ", err)
+	}
+	balance1Big, err := ethhelper.GetBalanceOf(common.HexToAddress(token1.Address), pairAddress, us.RestClient)
+	if err != nil {
+		log.Error("GetBalanceOf: ", err)
+	}
+	log.Infof("balance0 -- balance1 -- pool: %v -- %v -- %s ", balance0Big, balance1Big, pairAddress)
+	balance0, _ := new(big.Float).Quo(big.NewFloat(0).SetInt(balance0Big), new(big.Float).SetFloat64(math.Pow10(int(token0.Decimals)))).Float64()
+	balance1, _ := new(big.Float).Quo(big.NewFloat(0).SetInt(balance1Big), new(big.Float).SetFloat64(math.Pow10(int(token1.Decimals)))).Float64()
 
-	return pool, nil
+	pool.Assetvolumes = append(pool.Assetvolumes, dia.AssetVolume{Asset: token0, Volume: balance0, Index: uint8(0)})
+	pool.Assetvolumes = append(pool.Assetvolumes, dia.AssetVolume{Asset: token1, Volume: balance1, Index: uint8(1)})
+
+	// Determine USD liquidity
+	if balance0 > GLOBAL_NATIVE_LIQUIDITY_THRESHOLD && balance1 > GLOBAL_NATIVE_LIQUIDITY_THRESHOLD {
+		us.datastore.GetPoolLiquiditiesUSD(pool, priceCache)
+	}
 }
 
 // GetDecimals returns the decimals of the token with address @tokenAddress

pkg/dia/service/assetservice/source/velodrome.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/diadata-org/diadata/pkg/dia/scraper/exchange-scrapers/velodrome"
+	models "github.com/diadata-org/diadata/pkg/model"
 
 	"github.com/diadata-org/diadata/pkg/dia"
 	"github.com/diadata-org/diadata/pkg/utils"
@@ -24,22 +25,29 @@ type VelodromePair struct {
 
 const (
 	velodromeWaitMilliseconds = "500"
+	restDialOptimism          = ""
 )
 
 var velodromeExchangeFactoryContractAddress string
 
 type VelodromeAssetSource struct {
 	RestClient   *ethclient.Client
+	relDB        *models.RelDB
 	assetChannel chan dia.Asset
 	doneChannel  chan bool
+	exchange     dia.Exchange
 	blockchain   string
 	waitTime     int
 }
 
-func NewVelodromeAssetSource(exchange dia.Exchange) (uas *VelodromeAssetSource) {
+func NewVelodromeAssetSource(exchange dia.Exchange, relDB *models.RelDB) (uas *VelodromeAssetSource) {
 	switch exchange.Name {
 	case dia.VelodromeExchange:
-		uas = makeVelodromeAssetSource(exchange, exchange.RestAPI, velodromeWaitMilliseconds)
+		uas = makeVelodromeAssetSource(exchange, restDialOptimism, relDB, velodromeWaitMilliseconds)
+	case dia.AerodromeSlipstreamExchange:
+		uas = makeVelodromeAssetSource(exchange, restDialBase, relDB, velodromeWaitMilliseconds)
+	case dia.AerodromeV1Exchange:
+		uas = makeVelodromeAssetSource(exchange, restDialBase, relDB, velodromeWaitMilliseconds)
 	}
 
 	velodromeExchangeFactoryContractAddress = exchange.Contract
@@ -52,7 +60,7 @@ func NewVelodromeAssetSource(exchange dia.Exchange) (uas *VelodromeAssetSource)
 }
 
 // makeVelodromeAssetSource returns an asset source as used in NewVelodromeAssetSource.
-func makeVelodromeAssetSource(exchange dia.Exchange, restDial string, waitMilliseconds string) *VelodromeAssetSource {
+func makeVelodromeAssetSource(exchange dia.Exchange, restDial string, relDB *models.RelDB, waitMilliseconds string) *VelodromeAssetSource {
 	var (
 		restClient   *ethclient.Client
 		err          error
@@ -75,8 +83,10 @@ func makeVelodromeAssetSource(exchange dia.Exchange, restDial string, waitMillis
 	}
 	uas = &VelodromeAssetSource{
 		RestClient:   restClient,
+		relDB:        relDB,
 		assetChannel: assetChannel,
 		doneChannel:  doneChannel,
+		exchange:     exchange,
 		blockchain:   exchange.BlockChain.Name,
 		waitTime:     waitTime,
 	}
@@ -95,7 +105,7 @@ func (uas *VelodromeAssetSource) getNumPairs() (int, error) {
 	var contract *velodrome.PoolFactoryCaller
 	contract, err := velodrome.NewPoolFactoryCaller(common.HexToAddress(velodromeExchangeFactoryContractAddress), uas.RestClient)
 	if err != nil {
-		log.Error(err)
+		log.Error("getNumPairs: ", err)
 	}
 
 	numPairs, err := contract.AllPoolsLength(&bind.CallOpts{})
@@ -111,11 +121,18 @@ func (uas *VelodromeAssetSource) fetchAssets() {
 	log.Info("Found ", numPairs, " pairs")
 	checkMap := make(map[string]struct{})
 
-	for i := 0; i < numPairs; i++ {
+	_, index, err := uas.relDB.GetScraperIndex(uas.exchange.Name, dia.SCRAPER_TYPE_ASSETCOLLECTOR)
+	if err != nil {
+		log.Error("GetScraperIndex: ", err)
+	}
+	log.Info("Start at index ", index)
+
+	for index < int64(numPairs) {
+		log.Info("index: ", index)
 		time.Sleep(time.Duration(uas.waitTime) * time.Millisecond)
-		pair, err := uas.GetPairByID(int64(numPairs - 1 - i))
+		pair, err := uas.GetPairByID(int64(numPairs) - 1 - index)
 		if err != nil {
-			log.Errorln("Error getting pair with ID ", numPairs-1-i)
+			log.Errorln("Error getting pair with ID ", numPairs-1-int(index))
 		}
 		asset0 := pair.Token0
 		asset1 := pair.Token1
@@ -134,6 +151,7 @@ func (uas *VelodromeAssetSource) fetchAssets() {
 				uas.assetChannel <- asset1
 			}
 		}
+		index++
 	}
 	uas.doneChannel <- true
 }