Change materialization strategy for uniswap and nft trades (#1160)

This one should work ! I made sure the macros were updated as well, this should improve performance in terms of run/query time. I've checked that: * [ ] I tested the query on dune.com after compiling the model with dbt compile (compiled queries are written to the target directory) * [ ] I used "refs" to reference other models in this repo and "sources" to reference raw or decoded tables * [ ] the directory tree matches the pattern /sector/blockchain/ e.g. /tokens/ethereum * [ ] if adding a new model, I added a test * [ ] the filename is unique and ends with .sql * [ ] each file has only one view, table or function defined * [ ] column names are `lowercase_snake_cased`
duneanalytics · Jun 14, 2022 · 611c8e7 · 611c8e7
1 parent 7baea01
commit 611c8e7
Show file tree

Hide file tree

Showing 11 changed files with 73 additions and 65 deletions.
diff --git a/spellbook/macros/alter_table_properties.sql b/spellbook/macros/alter_table_properties.sql
@@ -56,7 +56,7 @@ ALTER VIEW balances_ethereum.erc721_latest SET TBLPROPERTIES('dune.public'='true
 {% endset %}
 
 {% set magiceden_trades %}
-ALTER VIEW magiceden.trades SET TBLPROPERTIES('dune.public'='true',
+ALTER TABLE magiceden.trades SET TBLPROPERTIES('dune.public'='true',
                                                     'dune.data_explorer.blockchains'='["solana"]',
                                                     'dune.data_explorer.category'='abstraction',
                                                     'dune.data_explorer.abstraction.type'='project',
@@ -65,7 +65,7 @@ ALTER VIEW magiceden.trades SET TBLPROPERTIES('dune.public'='true',
 {% endset %}
 
 {% set nft_trades %}
-ALTER VIEW nft.trades SET TBLPROPERTIES('dune.public'='true',
+ALTER TABLE nft.trades SET TBLPROPERTIES('dune.public'='true',
                                                     'dune.data_explorer.blockchains'='["ethereum","solana"]',
                                                     'dune.data_explorer.category'='abstraction',
                                                     'dune.data_explorer.abstraction.type'='sector',
@@ -83,7 +83,7 @@ ALTER VIEW opensea.active_traders_day SET TBLPROPERTIES('dune.public'='true',
 {% endset %}
 
 {% set opensea_trades %}
-ALTER VIEW opensea.trades SET TBLPROPERTIES('dune.public'='true',
+ALTER TABLE opensea.trades SET TBLPROPERTIES('dune.public'='true',
                                                     'dune.data_explorer.blockchains'='["ethereum","solana"]',
                                                     'dune.data_explorer.category'='abstraction',
                                                     'dune.data_explorer.abstraction.type'='project',
@@ -137,7 +137,7 @@ ALTER VIEW tokens_ethereum.nft SET TBLPROPERTIES('dune.public'='true',
 {% endset %}
 
 {% set uniswap_ethereum_trades %}
-ALTER VIEW uniswap_ethereum.trades SET TBLPROPERTIES('dune.public'='true',
+ALTER TABLE uniswap_ethereum.trades SET TBLPROPERTIES('dune.public'='true',
                                                     'dune.data_explorer.blockchains'='["ethereum"]',
                                                     'dune.data_explorer.category'='abstraction',
                                                     'dune.data_explorer.abstraction.type'='project',

diff --git a/spellbook/macros/optimize_tables.sql b/spellbook/macros/optimize_tables.sql
@@ -16,35 +16,31 @@ OPTIMIZE transfers_ethereum.erc721_agg_hour;
 OPTIMIZE transfers_ethereum.erc721_agg_day;
 {% endset %}
 
-{% set opensea_ethereum_trades %}
-OPTIMIZE opensea_ethereum.trades;
+{% set opensea_trades %}
+OPTIMIZE opensea.trades;
 {% endset %}
 
-{% set opensea_solana_trades %}
-OPTIMIZE opensea_solana.trades;
+{% set magiceden_trades %}
+OPTIMIZE magiceden.trades;
 {% endset %}
 
-{% set magiceden_solana_trades %}
-OPTIMIZE magiceden_solana.trades;
+{% set uniswap_ethereum_trades %}
+OPTIMIZE uniswap_ethereum.trades;
 {% endset %}
 
-{% set uniswap_v2_ethereum_trades %}
-OPTIMIZE uniswap_v2_ethereum.trades;
+{% set nft_trades %}
+OPTIMIZE nft.trades;
 {% endset %}
 
-{% set uniswap_v3_ethereum_trades %}
-OPTIMIZE uniswap_v3_ethereum.trades;
-{% endset %}
 
 {% do run_query(transfers_ethereum_erc20_agg_hour) %}
 {% do run_query(transfers_ethereum_erc20_agg_day) %}
 {% do run_query(transfers_ethereum_erc721_agg_hour) %}
 {% do run_query(transfers_ethereum_erc721_agg_day) %}
-{% do run_query(opensea_ethereum_trades) %}
-{% do run_query(opensea_solana_trades) %}
-{% do run_query(magiceden_solana_trades) %}
-{% do run_query(uniswap_v2_ethereum_trades) %}
-{% do run_query(uniswap_v3_ethereum_trades) %}
+{% do run_query(opensea_trades) %}
+{% do run_query(magiceden_trades) %}
+{% do run_query(uniswap_ethereum_trades) %}
+{% do run_query(nft_trades) %}
 
 {% do log("Tables Optimized", info=True) %}
 {%- else -%}

diff --git a/spellbook/models/magiceden/magiceden_trades.sql b/spellbook/models/magiceden/magiceden_trades.sql
@@ -1,10 +1,18 @@
 {{ config(
-        alias='trades'
+        alias='trades',
+        materialized ='incremental',
+        file_format ='delta',
+        incremental_strategy='merge',
+        unique_key='unique_trade_id'
         )
 }}
 
-SELECT blockchain, 'magiceden' as project, '' as version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM 
+SELECT blockchain, 'magiceden' as project, '' as version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, unique_trade_id FROM 
 (
-        SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id 
+        SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, unique_trade_id
         FROM {{ ref('magiceden_solana_trades') }}
-) 
+) 
+{% if is_incremental() %}
+-- this filter will only be applied on an incremental run
+WHERE block_time > now() - interval 2 days
+{% endif %} 
diff --git a/spellbook/models/magiceden/solana/magiceden_solana_trades.sql b/spellbook/models/magiceden/solana/magiceden_solana_trades.sql
@@ -9,25 +9,20 @@
 }}
 
 SELECT 
-  signatures[0] || id as unique_id,
+  signatures[0] || id as unique_trade_id,
   'solana' as blockchain,
   signatures[0] as tx_hash, 
   block_time,
   abs(post_balances[0] / 1e9 - pre_balances[0] / 1e9) * p.price AS amount_usd,
   abs(post_balances[0] / 1e9 - pre_balances[0] / 1e9) AS amount,
   p.symbol as token_symbol,
   p.contract_address as token_address,
-  account_keys[0] as traders,
-  id as trade_id
+  account_keys[0] as traders
 FROM {{ source('solana','transactions') }}
 LEFT JOIN {{ source('prices', 'usd') }} p 
   ON p.minute = date_trunc('minute', block_time)
   AND p.symbol = 'SOL'        
 WHERE (array_contains(account_keys, 'MEisE1HzehtrDpAAT8PnLHjpSSkRYakotTuJRPjTpo8') -- magic eden v1
        OR array_contains(account_keys, 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K'))  -- magic eden v2
 AND ARRAY_CONTAINS(log_messages, 'Program log: Instruction: ExecuteSale')
-AND block_time > '2021-09-01'
-{% if is_incremental() %}
--- this filter will only be applied on an incremental run
-AND block_time > now() - interval 2 days
-{% endif %} 
+AND block_time > '2021-09-01'
diff --git a/spellbook/models/nft/nft_trades.sql b/spellbook/models/nft/nft_trades.sql
@@ -1,13 +1,17 @@
 {{ config(
-        alias='trades'
+        alias='trades',
+        materialized ='incremental',
+        file_format ='delta',
+        incremental_strategy='merge',
+        unique_key='unique_trade_id'
         )
 }}
 
-SELECT blockchain, project, version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address,trade_id FROM 
-(SELECT blockchain, project, version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM {{ ref('opensea_trades') }} 
+SELECT blockchain, project, version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, unique_trade_id FROM 
+(SELECT blockchain, project, version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, unique_trade_id FROM {{ ref('opensea_trades') }} 
 UNION ALL
-SELECT blockchain, project, version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM {{ ref('magiceden_trades') }}) 
+SELECT blockchain, project, version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, unique_trade_id FROM {{ ref('magiceden_trades') }}) 
 {% if is_incremental() %}
 -- this filter will only be applied on an incremental run
-where block_time > (select max(block_time) from {{ this }})
+WHERE block_time > now() - interval 2 days
 {% endif %} 
diff --git a/spellbook/models/opensea/ethereum/opensea_ethereum_trades.sql b/spellbook/models/opensea/ethereum/opensea_ethereum_trades.sql
@@ -9,7 +9,7 @@
 }}
 
 SELECT
-evt_tx_hash || evt_index::string as unique_id,
+evt_tx_hash || evt_index::string as unique_trade_id,
 'ethereum' as blockchain,
 evt_tx_hash as tx_hash,
 evt_block_time as block_time,
@@ -19,8 +19,7 @@ om.price AS amount_raw,
 terc20.symbol as token_symbol,
 wam.token_address as token_address,
 maker,
-taker,
-evt_index as trade_id
+taker
 FROM
   {{ source('opensea_ethereum','wyvernexchange_evt_ordersmatched') }} om
   LEFT JOIN {{ ref('opensea_ethereum_wyvern_atomic_match') }} wam ON wam.tx_hash = om.evt_tx_hash
@@ -35,8 +34,4 @@ FROM
       *
     FROM
       {{ ref('opensea_ethereum_excluded_txns') }}
-  )
-  {% if is_incremental() %}
-  -- this filter will only be applied on an incremental run
-  AND evt_block_time > now() - interval 2 days
-  {% endif %} 
+  )
diff --git a/spellbook/models/opensea/opensea_trades.sql b/spellbook/models/opensea/opensea_trades.sql
@@ -1,9 +1,18 @@
 {{ config(
-        alias='trades'
+        alias='trades',
+        materialized ='incremental',
+        file_format ='delta',
+        incremental_strategy='merge',
+        unique_key='unique_trade_id'
         )
 }}
 
-SELECT blockchain, 'opensea' as project, 'v1' as version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM 
-(SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM {{ ref('opensea_ethereum_trades') }} 
+SELECT blockchain, 'opensea' as project, 'v1' as version, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, unique_trade_id FROM 
+(SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, unique_trade_id FROM {{ ref('opensea_ethereum_trades') }} 
 UNION ALL
-SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, trade_id FROM {{ ref('opensea_solana_trades') }}) 
+SELECT blockchain, tx_hash, block_time, amount_usd, amount, token_symbol, token_address, unique_trade_id FROM {{ ref('opensea_solana_trades') }}) 
+{% if is_incremental() %}
+-- this filter will only be applied on an incremental run
+WHERE block_time > now() - interval 2 days
+{% endif %} 
+
diff --git a/spellbook/models/opensea/solana/opensea_solana_trades.sql b/spellbook/models/opensea/solana/opensea_solana_trades.sql
@@ -9,25 +9,20 @@
 }}
 
 SELECT 
-  signatures[0] || id as unique_id,
+  signatures[0] || id as unique_trade_id,
   'solana' as blockchain,
   signatures[0] as tx_hash, 
   block_time,
   abs(post_balances[0] / 1e9 - pre_balances[0] / 1e9) * p.price AS amount_usd,
   abs(post_balances[0] / 1e9 - pre_balances[0] / 1e9) AS amount,
   p.symbol as token_symbol,
   p.contract_address as token_address,
-  account_keys[0] as traders,
-  id as trade_id
+  account_keys[0] as traders
 FROM {{ source('solana','transactions') }}
 LEFT JOIN {{ source('prices', 'usd') }} p 
   ON p.minute = date_trunc('minute', block_time)
   AND p.symbol = 'SOL'
 WHERE (array_contains(account_keys, '3o9d13qUvEuuauhFrVom1vuCzgNsJifeaBYDPquaT73Y')
        OR array_contains(account_keys, 'pAHAKoTJsAAe2ZcvTZUxoYzuygVAFAmbYmJYdWT886r'))
 AND block_time > '2022-04-06'
-AND ARRAY_CONTAINS(log_messages, 'Program log: Instruction: ExecuteSale')
-{% if is_incremental() %}
--- this filter will only be applied on an incremental run
-AND block_time > now() - interval 2 days
-{% endif %} 
+AND ARRAY_CONTAINS(log_messages, 'Program log: Instruction: ExecuteSale')
diff --git a/spellbook/models/uniswap/ethereum/uniswap_ethereum_trades.sql b/spellbook/models/uniswap/ethereum/uniswap_ethereum_trades.sql
@@ -1,12 +1,20 @@
  {{
   config(
-        alias='trades'
+        alias='trades',        
+        materialized ='incremental',
+        file_format ='delta',
+        incremental_strategy='merge',
+        unique_key='unique_trade_id'
   )
 }}
 
 SELECT blockchain, project, version, block_time, token_a_symbol, token_b_symbol, 
        token_a_amount, token_b_amount, trader_a, trader_b, usd_amount, token_a_address, 
-       token_b_address, exchange_contract_address, tx_hash, tx_from, tx_to, trade_id 
+       token_b_address, exchange_contract_address, tx_hash, tx_from, tx_to, unique_trade_id
 FROM (SELECT * FROM {{ ref('uniswap_v2_ethereum_trades') }} 
 UNION ALL
 SELECT * FROM {{ ref('uniswap_v3_ethereum_trades') }}) 
+{% if is_incremental() %}
+-- this filter will only be applied on an incremental run
+WHERE block_time > now() - interval 2 days
+{% endif %} 
diff --git a/spellbook/models/uniswap/ethereum/uniswap_v2_ethereum_trades.sql b/spellbook/models/uniswap/ethereum/uniswap_v2_ethereum_trades.sql
@@ -10,7 +10,7 @@
 }}
 
 SELECT
-    tx_hash || evt_index::string as unique_id,
+    tx_hash || evt_index::string as unique_trade_id,
     'ethereum' as blockchain,
     'uniswap' as project, 
     'v2' as version,
@@ -31,8 +31,7 @@ SELECT
     exchange_contract_address,
     tx_hash,
     tx.from as tx_from,
-    tx.to as tx_to,
-    evt_index as trade_id
+    tx.to as tx_to
 FROM (
     --Uniswap v2
     SELECT

diff --git a/spellbook/models/uniswap/ethereum/uniswap_v3_ethereum_trades.sql b/spellbook/models/uniswap/ethereum/uniswap_v3_ethereum_trades.sql
@@ -7,7 +7,7 @@
 }}
 
 SELECT
-    tx_hash || evt_index::string as unique_id,
+    tx_hash || evt_index::string as unique_trade_id,
     'ethereum' as blockchain,
     'uniswap' as project, 
     'v3' as version,
@@ -28,8 +28,7 @@ SELECT
     exchange_contract_address,
     tx_hash,
     tx.from as tx_from,
-    tx.to as tx_to,
-    evt_index as trade_id
+    tx.to as tx_to
     FROM (--Uniswap v3
     SELECT
     t.evt_block_time AS block_time,