From 24358e334b7f5891c6661b08befcf17192c5a6b4 Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 31 Oct 2024 17:08:29 -0600 Subject: [PATCH 01/11] add collateral balance model --- ...p_collateral_balances_arbitrum_mainnet.sql | 99 +++++++++++++++++++ .../marts/arbitrum/mainnet/perp/schema.yml | 35 +++++++ .../arbitrum_mainnet_reward_distributors.csv | 18 ++-- 3 files changed, 143 insertions(+), 9 deletions(-) create mode 100644 transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql new file mode 100644 index 00000000..cb6ec4f8 --- /dev/null +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql @@ -0,0 +1,99 @@ +WITH synths AS ( + SELECT + synth_market_id AS collateral_id, + synth_token_address + FROM + {{ ref('spot_synth_registered_arbitrum_mainnet') }} +), +transfers AS ( + SELECT + cm.block_number, + cm.block_timestamp AS ts, + cm.collateral_id, + synths.synth_token_address, + cm.amount_delta / 1e18 AS amount_delta + FROM + {{ ref('perp_collateral_modified_arbitrum_mainnet') }} + cm + JOIN synths + ON cm.collateral_id = synths.collateral_id +), +liq_tx AS ( + SELECT + DISTINCT transaction_hash + FROM + {{ ref('perp_account_liquidation_attempt_arbitrum_mainnet') }} +), +distributors AS ( + SELECT + CAST( + rd.distributor_address AS text + ) AS distributor_address, + CAST( + rd.token_symbol AS text + ) AS token_symbol, + rd.synth_token_address, + synths.collateral_id + FROM + {{ ref('arbitrum_mainnet_reward_distributors') }} + rd + JOIN synths + ON synths.synth_token_address = rd.synth_token_address +), +liquidations AS ( + SELECT + block_number, + block_timestamp AS ts,- amount / 1e18 AS amount_delta, + collateral_type, + token_symbol, + synth_token_address, + distributors.collateral_id + FROM + prod_raw_arbitrum_mainnet.core_rewards_distributed_arbitrum_mainnet rd + JOIN liq_tx + ON rd.transaction_hash = liq_tx.transaction_hash + JOIN distributors + ON rd.distributor = distributors.distributor_address +), +net_transfers AS ( + SELECT + ts, + event_type, + collateral_id, + synth_token_address, + amount_delta, + SUM(amount_delta) over ( + PARTITION BY collateral_id + ORDER BY + ts + ) AS net_amount + FROM + ( + SELECT + ts, + collateral_id, + synth_token_address, + amount_delta, + 'transfer' AS event_type + FROM + transfers + UNION ALL + SELECT + ts, + collateral_id, + synth_token_address, + amount_delta, + 'liquidation' AS event_type + FROM + liquidations + ) A +) +SELECT + ts, + collateral_id, + synth_token_address, + event_type, + amount_delta, + net_amount +FROM + net_transfers diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml index 634c382f..0ae96a74 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml @@ -662,3 +662,38 @@ models: data_type: text tests: - not_null + - name: fct_perp_collateral_balances_arbitrum_mainnet + columns: + - name: ts + description: "Block timestamp" + data_type: timestamp with time zone + tests: + - not_null + - name: collateral_id + description: "Market ID of the collateral, from the spot market" + data_type: integer + tests: + - not_null + - name: synth_token_address + description: "Address of the synth token" + data_type: text + tests: + - not_null + - name: event_type + description: "Description of the event, e.g. 'transfer' or 'liquidation'" + data_type: text + tests: + - not_null + - name: amount_delta + description: "Amount of the collateral token that was transferred" + data_type: numeric + tests: + - not_null + - name: net_amount + description: "Balance of the collateral token after the transfer" + data_type: numeric + tests: + - not_null + - dbt_utils.accepted_range: + min_value: 0 + inclusive: true diff --git a/transformers/synthetix/seeds/reward_distributors/arbitrum_mainnet_reward_distributors.csv b/transformers/synthetix/seeds/reward_distributors/arbitrum_mainnet_reward_distributors.csv index 0c337580..aaffcdbb 100644 --- a/transformers/synthetix/seeds/reward_distributors/arbitrum_mainnet_reward_distributors.csv +++ b/transformers/synthetix/seeds/reward_distributors/arbitrum_mainnet_reward_distributors.csv @@ -1,9 +1,9 @@ -distributor_address,token_symbol -0x9Ac841f5716FDe2AbD5e966695B8BBCC29d7CeE5,ARB -0x8B6D6416017de3d1dCac4158Fe8a401C6B951fA8,ARB -0x806b9D4b0a56b6D1d9CaCb60AEe95E8d885BD55A,ARB -0x5eff594E10aCb20919a577794C4e9B4bb1221F17,ARB -0x28B69C6a5DDd5e370114B164E7A9CaBf051B6B87,BTC -0xD237F237356e456214Ad5b57e0A962d632F1F3C5,WETH -0xB363932c7C56f2B6b684abbD63cc4515f015416a,SOL -0x8CB5d51099933E04BAa1a32dE3AdED3DaC35f13B,USDe \ No newline at end of file +distributor_address,token_symbol,synth_token_address +0x9Ac841f5716FDe2AbD5e966695B8BBCC29d7CeE5,ARB, +0x8B6D6416017de3d1dCac4158Fe8a401C6B951fA8,ARB, +0x806b9D4b0a56b6D1d9CaCb60AEe95E8d885BD55A,ARB, +0x5eff594E10aCb20919a577794C4e9B4bb1221F17,ARB, +0x28B69C6a5DDd5e370114B164E7A9CaBf051B6B87,BTC,0xFA86aB03412Ab63Fea238d43D1E839c4F7A80232 +0xD237F237356e456214Ad5b57e0A962d632F1F3C5,WETH,0x3Ec3FedA50b718b5A9ff387F93EeA7841D795B1E +0xB363932c7C56f2B6b684abbD63cc4515f015416a,SOL,0xb74Da9FE2F96B9E0a5f4A3cf0b92dd2bEC617124 +0x8CB5d51099933E04BAa1a32dE3AdED3DaC35f13B,USDe,0xE3eE09c200584228F7C45d50E12BcC3fb65c19Ca \ No newline at end of file From 8e3ed0310cbe7a84bde37407dce7e4e4c505cb5a Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 31 Oct 2024 17:28:00 -0600 Subject: [PATCH 02/11] add to model --- ...p_collateral_balances_arbitrum_mainnet.sql | 32 ++++++++++++++++--- .../marts/arbitrum/mainnet/perp/schema.yml | 17 ++++++++-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql index cb6ec4f8..393269ce 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql @@ -9,9 +9,13 @@ transfers AS ( SELECT cm.block_number, cm.block_timestamp AS ts, + cm.transaction_hash, cm.collateral_id, synths.synth_token_address, - cm.amount_delta / 1e18 AS amount_delta + account_id, + {{ convert_wei( + 'cm.amount_delta' + ) }} AS amount_delta FROM {{ ref('perp_collateral_modified_arbitrum_mainnet') }} cm @@ -20,7 +24,8 @@ transfers AS ( ), liq_tx AS ( SELECT - DISTINCT transaction_hash + DISTINCT transaction_hash, + account_id FROM {{ ref('perp_account_liquidation_attempt_arbitrum_mainnet') }} ), @@ -44,9 +49,11 @@ liquidations AS ( SELECT block_number, block_timestamp AS ts,- amount / 1e18 AS amount_delta, + liq_tx.transaction_hash, collateral_type, token_symbol, synth_token_address, + account_id, distributors.collateral_id FROM prod_raw_arbitrum_mainnet.core_rewards_distributed_arbitrum_mainnet rd @@ -58,21 +65,31 @@ liquidations AS ( net_transfers AS ( SELECT ts, + transaction_hash, event_type, collateral_id, synth_token_address, + account_id, amount_delta, + SUM(amount_delta) over ( + PARTITION BY account_id, + collateral_id + ORDER BY + ts + ) AS account_balance, SUM(amount_delta) over ( PARTITION BY collateral_id ORDER BY ts - ) AS net_amount + ) AS total_balance FROM ( SELECT ts, + transaction_hash, collateral_id, synth_token_address, + account_id, amount_delta, 'transfer' AS event_type FROM @@ -80,8 +97,10 @@ net_transfers AS ( UNION ALL SELECT ts, + transaction_hash, collateral_id, synth_token_address, + account_id, amount_delta, 'liquidation' AS event_type FROM @@ -90,10 +109,13 @@ net_transfers AS ( ) SELECT ts, + transaction_hash, + event_type, collateral_id, synth_token_address, - event_type, + account_id, amount_delta, - net_amount + account_balance, + total_balance FROM net_transfers diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml index 0ae96a74..00d3bdfa 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml @@ -669,6 +669,11 @@ models: data_type: timestamp with time zone tests: - not_null + - name: transaction_hash + description: "Transaction hash" + data_type: text + tests: + - not_null - name: collateral_id description: "Market ID of the collateral, from the spot market" data_type: integer @@ -689,8 +694,16 @@ models: data_type: numeric tests: - not_null - - name: net_amount - description: "Balance of the collateral token after the transfer" + - name: account_balance + description: "Balance of the collateral token in the account after the transfer" + data_type: numeric + tests: + - not_null + - dbt_utils.accepted_range: + min_value: 0 + inclusive: true + - name: total_balance + description: "Total balance of the collateral token in the perps system after the transfer" data_type: numeric tests: - not_null From 73df10ca972d0ef04e0560cc66ebdfe34352cb3b Mon Sep 17 00:00:00 2001 From: Troy Date: Fri, 1 Nov 2024 13:50:46 -0600 Subject: [PATCH 03/11] add synths --- Makefile | 3 ++ docker-compose.yml | 2 + transformers/requirements.txt | 1 + ...p_collateral_balances_arbitrum_mainnet.sql | 39 ++++++++++------ transformers/synthetix/scripts/get_synths.py | 46 +++++++++++++++++++ .../seeds/synths/arbitrum_mainnet_synths.csv | 7 +++ .../seeds/synths/arbitrum_sepolia_synths.csv | 8 ++++ .../seeds/synths/base_mainnet_synths.csv | 4 ++ .../seeds/synths/base_sepolia_synths.csv | 8 ++++ .../seeds/synths/eth_mainnet_synths.csv | 3 ++ 10 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 transformers/synthetix/scripts/get_synths.py create mode 100644 transformers/synthetix/seeds/synths/arbitrum_mainnet_synths.csv create mode 100644 transformers/synthetix/seeds/synths/arbitrum_sepolia_synths.csv create mode 100644 transformers/synthetix/seeds/synths/base_mainnet_synths.csv create mode 100644 transformers/synthetix/seeds/synths/base_sepolia_synths.csv create mode 100644 transformers/synthetix/seeds/synths/eth_mainnet_synths.csv diff --git a/Makefile b/Makefile index 2429b5db..d4cb4492 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,9 @@ extract: docker compose run extractors python main.py configs/arbitrum_mainnet.yaml docker compose run extractors python main.py configs/arbitrum_sepolia.yaml +synths: + docker compose run transformer python scripts/get_synths.py + wrap: docker compose run transformer python scripts/wrap_tables.py diff --git a/docker-compose.yml b/docker-compose.yml index 1c6c3fd4..55256244 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -170,6 +170,8 @@ services: context: ./transformers depends_on: - db + env_file: + - .env environment: PG_PASSWORD: $PG_PASSWORD volumes: diff --git a/transformers/requirements.txt b/transformers/requirements.txt index 0f44ee95..9fda24fe 100644 --- a/transformers/requirements.txt +++ b/transformers/requirements.txt @@ -3,3 +3,4 @@ pyarrow psycopg2-binary dbt-postgres sqlfluff +synthetix==0.1.21 \ No newline at end of file diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql index 393269ce..537adfe4 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql @@ -64,23 +64,28 @@ liquidations AS ( ), net_transfers AS ( SELECT - ts, - transaction_hash, - event_type, - collateral_id, - synth_token_address, - account_id, - amount_delta, - SUM(amount_delta) over ( - PARTITION BY account_id, - collateral_id + events.ts, + events.transaction_hash, + events.event_type, + events.collateral_id, + events.synth_token_address, + synths.synth_symbol, + events.account_id, + events.amount_delta, + SUM( + events.amount_delta + ) over ( + PARTITION BY events.account_id, + events.collateral_id ORDER BY - ts + events.ts ) AS account_balance, - SUM(amount_delta) over ( - PARTITION BY collateral_id + SUM( + events.amount_delta + ) over ( + PARTITION BY events.collateral_id ORDER BY - ts + events.ts ) AS total_balance FROM ( @@ -105,7 +110,10 @@ net_transfers AS ( 'liquidation' AS event_type FROM liquidations - ) A + ) events + JOIN {{ ref('arbitrum_mainnet_synths') }} + synths + ON events.collateral_id = synths.synth_market_id ) SELECT ts, @@ -113,6 +121,7 @@ SELECT event_type, collateral_id, synth_token_address, + synth_symbol, account_id, amount_delta, account_balance, diff --git a/transformers/synthetix/scripts/get_synths.py b/transformers/synthetix/scripts/get_synths.py new file mode 100644 index 00000000..fa0b941e --- /dev/null +++ b/transformers/synthetix/scripts/get_synths.py @@ -0,0 +1,46 @@ +import os +import pandas as pd +from synthetix import Synthetix + +NETWORK_IDS = { + 1: "eth_mainnet", + # 11155111: "eth_sepolia", + 8453: "base_mainnet", + 84532: "base_sepolia", + 42161: "arbitrum_mainnet", + 421614: "arbitrum_sepolia", +} + + +def get_synths(network_id): + network_name = NETWORK_IDS[network_id] + + # start the sdk + snx = Synthetix(provider_rpc=os.getenv(f"NETWORK_{network_id}_RPC")) + + # get the synths + synths = [] + for idx in snx.spot.markets_by_id: + market = snx.spot.markets_by_id[idx] + contract = market["contract"] + + synth_name = contract.functions.name().call() + token_symbol = market["symbol"] if idx != 0 else "sUSD" + synths.append( + { + "synth_market_id": idx, + "synth_token_address": contract.address, + "synth_name": synth_name, + "synth_symbol": market["market_name"], + "token_symbol": token_symbol, + } + ) + + # write it out + df_synths = pd.DataFrame(synths) + df_synths.to_csv(f"./seeds/synths/{network_name}_synths.csv", index=False) + + +if __name__ == "__main__": + for network_id in NETWORK_IDS: + get_synths(network_id) diff --git a/transformers/synthetix/seeds/synths/arbitrum_mainnet_synths.csv b/transformers/synthetix/seeds/synths/arbitrum_mainnet_synths.csv new file mode 100644 index 00000000..78d1eaf2 --- /dev/null +++ b/transformers/synthetix/seeds/synths/arbitrum_mainnet_synths.csv @@ -0,0 +1,7 @@ +synth_market_id,synth_token_address,synth_name,synth_symbol,token_symbol +0,0xb2F30A7C980f052f02563fb518dcc39e6bf38175,Synthetix USD,sUSD,sUSD +2,0xE81Be4495f138FAE5846d21AC2cA822BEf452365,Synthetic USD Coin,sUSDC,USDC +3,0xFA86aB03412Ab63Fea238d43D1E839c4F7A80232,Synthetic Bitcoin,stBTC,tBTC +4,0x3Ec3FedA50b718b5A9ff387F93EeA7841D795B1E,Synthetic Ether,sETH,ETH +5,0xE3eE09c200584228F7C45d50E12BcC3fb65c19Ca,Synthetic USDe,sUSDe,USDe +6,0x7301a8DBd293b85A06726aE12E433a829ba3B871,Synthetic Wormhole SOL,swSOL,wSOL diff --git a/transformers/synthetix/seeds/synths/arbitrum_sepolia_synths.csv b/transformers/synthetix/seeds/synths/arbitrum_sepolia_synths.csv new file mode 100644 index 00000000..6b0a3891 --- /dev/null +++ b/transformers/synthetix/seeds/synths/arbitrum_sepolia_synths.csv @@ -0,0 +1,8 @@ +synth_market_id,synth_token_address,synth_name,synth_symbol,token_symbol +0,0xe487Ad4291019b33e2230F8E2FB1fb6490325260,Synthetix USD,sUSD,sUSD +1,0x88DCcaae7872377f80c59cf3c04865741fDbaffF,Synthetic DAI,sDAI,DAI +2,0xB3A0E9e0699b82b1F33A03fFf38e72513c2Dc933,Synthetic USD Coin,sUSDC,USDC +4,0x726cFb7f07231D8dBfE993c3A8b09D933CaeE78b,Synthetic Bitcoin,sBTC,BTC +5,0x711d4F22CC39223a7273E45537c9d371E66A8a40,Synthetic Ether,sETH,ETH +7,0xD58A253022fE19693abC61e041C88F32b51C7Cbf,Synthetic USDe,sUSDe,USDe +8,0xDE0a47cf02bd6c4cf5Dc30A059656c26A10D01B8,Synthetic SOL,sSOL,SOL diff --git a/transformers/synthetix/seeds/synths/base_mainnet_synths.csv b/transformers/synthetix/seeds/synths/base_mainnet_synths.csv new file mode 100644 index 00000000..3e72add4 --- /dev/null +++ b/transformers/synthetix/seeds/synths/base_mainnet_synths.csv @@ -0,0 +1,4 @@ +synth_market_id,synth_token_address,synth_name,synth_symbol,token_symbol +0,0x09d51516F38980035153a554c26Df3C6f51a23C3,Synthetic USD Token v3,sUSD,sUSD +1,0xC74eA762cF06c9151cE074E6a569a5945b6302E7,Synthetic USD Coin,sUSDC,USDC +3,0x729Ef31D86d31440ecBF49f27F7cD7c16c6616d2,Synthetic Static aUSDC,sStataUSDC,StataUSDC diff --git a/transformers/synthetix/seeds/synths/base_sepolia_synths.csv b/transformers/synthetix/seeds/synths/base_sepolia_synths.csv new file mode 100644 index 00000000..da6d9596 --- /dev/null +++ b/transformers/synthetix/seeds/synths/base_sepolia_synths.csv @@ -0,0 +1,8 @@ +synth_market_id,synth_token_address,synth_name,synth_symbol,token_symbol +0,0x682f0d17feDC62b2a0B91f8992243Bf44cAfeaaE,Synthetic USD Token v3,sUSD,sUSD +1,0x8069c44244e72443722cfb22DcE5492cba239d39,Synthetic USD Coin,sUSDC,USDC +4,0xB94c6E4f5162717c6fAb1Eeab8f0296307F91528,Synthetic Static aUSDC,sStataUSDC,StataUSDC +7,0x410EecB4b4CF7175352a472572492C1c9997a5e8,Synthetic Coinbase Wrapped BTC,scbBTC,cbBTC +8,0x1c6dfe3205334Fece6a9169c88bF698Ed4370107,Synthetic Coinbase Wrapped Staked ETH,scbETH,cbETH +9,0x86B35F1b900B15C98049f68f4248815518e71985,Synthetic Wrapped ETH,sWETH,WETH +10,0x5dc2592d23f72833c559ACB35c7122995EA80486,Synthetic Lido Wrapped Staked ETH,swstETH,wstETH diff --git a/transformers/synthetix/seeds/synths/eth_mainnet_synths.csv b/transformers/synthetix/seeds/synths/eth_mainnet_synths.csv new file mode 100644 index 00000000..a5e47db0 --- /dev/null +++ b/transformers/synthetix/seeds/synths/eth_mainnet_synths.csv @@ -0,0 +1,3 @@ +synth_market_id,synth_token_address,synth_name,synth_symbol,token_symbol +0,0xb2F30A7C980f052f02563fb518dcc39e6bf38175,Synthetic USD Token v3,sUSD,sUSD +2,0x3f3eD1002F55121b8213182f09d4BE544E08B9F9,Synthetic Ethena USD,snxUSDe,nxUSDe From 5a43f089f96ff49dec06ec0e8ee0125cf9e6eb6d Mon Sep 17 00:00:00 2001 From: Troy Date: Fri, 1 Nov 2024 13:56:51 -0600 Subject: [PATCH 04/11] update schema and add arb sepolia --- .../marts/arbitrum/mainnet/perp/schema.yml | 5 + ...p_collateral_balances_arbitrum_sepolia.sql | 130 ++++++++++++++++++ .../marts/arbitrum/sepolia/perp/schema.yml | 53 +++++++ .../arbitrum_sepolia_reward_distributors.csv | 6 +- 4 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml index 00d3bdfa..25aaba94 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml @@ -684,6 +684,11 @@ models: data_type: text tests: - not_null + - name: synth_symbol + description: "Symbol of the synth token" + data_type: text + tests: + - not_null - name: event_type description: "Description of the event, e.g. 'transfer' or 'liquidation'" data_type: text diff --git a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql new file mode 100644 index 00000000..cdee3a83 --- /dev/null +++ b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql @@ -0,0 +1,130 @@ +WITH synths AS ( + SELECT + synth_market_id AS collateral_id, + synth_token_address + FROM + {{ ref('spot_synth_registered_arbitrum_sepolia') }} +), +transfers AS ( + SELECT + cm.block_number, + cm.block_timestamp AS ts, + cm.transaction_hash, + cm.collateral_id, + synths.synth_token_address, + account_id, + {{ convert_wei( + 'cm.amount_delta' + ) }} AS amount_delta + FROM + {{ ref('perp_collateral_modified_arbitrum_sepolia') }} + cm + JOIN synths + ON cm.collateral_id = synths.collateral_id +), +liq_tx AS ( + SELECT + DISTINCT transaction_hash, + account_id + FROM + {{ ref('perp_account_liquidation_attempt_arbitrum_sepolia') }} +), +distributors AS ( + SELECT + CAST( + rd.distributor_address AS text + ) AS distributor_address, + CAST( + rd.token_symbol AS text + ) AS token_symbol, + rd.synth_token_address, + synths.collateral_id + FROM + {{ ref('arbitrum_sepolia_reward_distributors') }} + rd + JOIN synths + ON synths.synth_token_address = rd.synth_token_address +), +liquidations AS ( + SELECT + block_number, + block_timestamp AS ts,- amount / 1e18 AS amount_delta, + liq_tx.transaction_hash, + collateral_type, + token_symbol, + synth_token_address, + account_id, + distributors.collateral_id + FROM + prod_raw_arbitrum_sepolia.core_rewards_distributed_arbitrum_sepolia rd + JOIN liq_tx + ON rd.transaction_hash = liq_tx.transaction_hash + JOIN distributors + ON rd.distributor = distributors.distributor_address +), +net_transfers AS ( + SELECT + events.ts, + events.transaction_hash, + events.event_type, + events.collateral_id, + events.synth_token_address, + synths.synth_symbol, + events.account_id, + events.amount_delta, + SUM( + events.amount_delta + ) over ( + PARTITION BY events.account_id, + events.collateral_id + ORDER BY + events.ts + ) AS account_balance, + SUM( + events.amount_delta + ) over ( + PARTITION BY events.collateral_id + ORDER BY + events.ts + ) AS total_balance + FROM + ( + SELECT + ts, + transaction_hash, + collateral_id, + synth_token_address, + account_id, + amount_delta, + 'transfer' AS event_type + FROM + transfers + UNION ALL + SELECT + ts, + transaction_hash, + collateral_id, + synth_token_address, + account_id, + amount_delta, + 'liquidation' AS event_type + FROM + liquidations + ) events + JOIN {{ ref('arbitrum_sepolia_synths') }} + synths + ON events.collateral_id = synths.synth_market_id +) +SELECT + ts, + transaction_hash, + event_type, + collateral_id, + synth_token_address, + synth_symbol, + account_id, + amount_delta, + account_balance, + total_balance +FROM + net_transfers diff --git a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml index 1b624a35..dd67c02c 100644 --- a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml +++ b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml @@ -662,3 +662,56 @@ models: data_type: text tests: - not_null + - name: fct_perp_collateral_balances_arbitrum_mainnet + columns: + - name: ts + description: "Block timestamp" + data_type: timestamp with time zone + tests: + - not_null + - name: transaction_hash + description: "Transaction hash" + data_type: text + tests: + - not_null + - name: collateral_id + description: "Market ID of the collateral, from the spot market" + data_type: integer + tests: + - not_null + - name: synth_token_address + description: "Address of the synth token" + data_type: text + tests: + - not_null + - name: synth_symbol + description: "Symbol of the synth token" + data_type: text + tests: + - not_null + - name: event_type + description: "Description of the event, e.g. 'transfer' or 'liquidation'" + data_type: text + tests: + - not_null + - name: amount_delta + description: "Amount of the collateral token that was transferred" + data_type: numeric + tests: + - not_null + - name: account_balance + description: "Balance of the collateral token in the account after the transfer" + data_type: numeric + tests: + - not_null + - dbt_utils.accepted_range: + min_value: 0 + inclusive: true + - name: total_balance + description: "Total balance of the collateral token in the perps system after the transfer" + data_type: numeric + tests: + - not_null + - dbt_utils.accepted_range: + min_value: 0 + inclusive: true diff --git a/transformers/synthetix/seeds/reward_distributors/arbitrum_sepolia_reward_distributors.csv b/transformers/synthetix/seeds/reward_distributors/arbitrum_sepolia_reward_distributors.csv index b09f7b75..2f6f666c 100644 --- a/transformers/synthetix/seeds/reward_distributors/arbitrum_sepolia_reward_distributors.csv +++ b/transformers/synthetix/seeds/reward_distributors/arbitrum_sepolia_reward_distributors.csv @@ -1,8 +1,8 @@ -distributor_address,token_symbol +distributor_address,token_symbol,synth_token_address 0x8Db7E7E3f00c2Fd011Bb715935d33BfA0570cDB6,ARB 0x26c9B83fA781e93Ec4cA5fBAC07AC82135872512,ARB 0x4aFF595044Ac6844151481B031D9A2E291aE37fD,ARB 0x3974935755a50295b727Ab40ae144BE29391d818,ARB 0xc555AfdCa6FbeF64fbE9feD62E27Bd98F8a686D3,ARB -0xb1dCa5430db838C1D43e349AAe8b04Bc54a1416d,BTC -0xeCbCb88bff2f5f24173793cf86DF08f852eD4f2A,WETH +0xb1dCa5430db838C1D43e349AAe8b04Bc54a1416d,BTC,0x726cFb7f07231D8dBfE993c3A8b09D933CaeE78b +0xeCbCb88bff2f5f24173793cf86DF08f852eD4f2A,WETH,0x711d4F22CC39223a7273E45537c9d371E66A8a40 From 3c801f8530256060b10db3eeb84bd637b3777884 Mon Sep 17 00:00:00 2001 From: Troy Date: Fri, 1 Nov 2024 14:08:11 -0600 Subject: [PATCH 05/11] fix name --- .../synthetix/models/marts/arbitrum/sepolia/perp/schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml index dd67c02c..e20cb81b 100644 --- a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml +++ b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml @@ -662,7 +662,7 @@ models: data_type: text tests: - not_null - - name: fct_perp_collateral_balances_arbitrum_mainnet + - name: fct_perp_collateral_balances_arbitrum_sepolia columns: - name: ts description: "Block timestamp" From 581220857abfe795bf80fd2e4e497c0db50204d4 Mon Sep 17 00:00:00 2001 From: Troy Date: Fri, 1 Nov 2024 14:20:34 -0600 Subject: [PATCH 06/11] linting --- ...p_collateral_balances_arbitrum_mainnet.sql | 125 ++++++++++-------- ...p_collateral_balances_arbitrum_sepolia.sql | 125 ++++++++++-------- 2 files changed, 134 insertions(+), 116 deletions(-) diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql index 537adfe4..0573b5ef 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql @@ -1,69 +1,75 @@ -WITH synths AS ( - SELECT - synth_market_id AS collateral_id, +with synths as ( + select + synth_market_id as collateral_id, synth_token_address - FROM + from {{ ref('spot_synth_registered_arbitrum_mainnet') }} ), -transfers AS ( - SELECT + +transfers as ( + select cm.block_number, - cm.block_timestamp AS ts, + cm.block_timestamp as ts, cm.transaction_hash, cm.collateral_id, synths.synth_token_address, account_id, {{ convert_wei( 'cm.amount_delta' - ) }} AS amount_delta - FROM + ) }} as amount_delta + from {{ ref('perp_collateral_modified_arbitrum_mainnet') }} - cm - JOIN synths - ON cm.collateral_id = synths.collateral_id + as cm + inner join synths + on cm.collateral_id = synths.collateral_id ), -liq_tx AS ( - SELECT - DISTINCT transaction_hash, + +liq_tx as ( + select distinct + transaction_hash, account_id - FROM + from {{ ref('perp_account_liquidation_attempt_arbitrum_mainnet') }} ), -distributors AS ( - SELECT + +distributors as ( + select CAST( - rd.distributor_address AS text - ) AS distributor_address, + rd.distributor_address as text + ) as distributor_address, CAST( - rd.token_symbol AS text - ) AS token_symbol, + rd.token_symbol as text + ) as token_symbol, rd.synth_token_address, synths.collateral_id - FROM + from {{ ref('arbitrum_mainnet_reward_distributors') }} - rd - JOIN synths - ON synths.synth_token_address = rd.synth_token_address + as rd + inner join synths + on rd.synth_token_address = synths.synth_token_address ), -liquidations AS ( - SELECT + +liquidations as ( + select block_number, - block_timestamp AS ts,- amount / 1e18 AS amount_delta, + block_timestamp as ts, + -amount / 1e18 as amount_delta, liq_tx.transaction_hash, collateral_type, token_symbol, synth_token_address, account_id, distributors.collateral_id - FROM - prod_raw_arbitrum_mainnet.core_rewards_distributed_arbitrum_mainnet rd - JOIN liq_tx - ON rd.transaction_hash = liq_tx.transaction_hash - JOIN distributors - ON rd.distributor = distributors.distributor_address + from + prod_raw_arbitrum_mainnet.core_rewards_distributed_arbitrum_mainnet as rd + inner join liq_tx + on rd.transaction_hash = liq_tx.transaction_hash + inner join distributors + on rd.distributor = distributors.distributor_address ), -net_transfers AS ( - SELECT + +net_transfers as ( + select events.ts, events.transaction_hash, events.event_type, @@ -75,47 +81,50 @@ net_transfers AS ( SUM( events.amount_delta ) over ( - PARTITION BY events.account_id, - events.collateral_id - ORDER BY + partition by + events.account_id, + events.collateral_id + order by events.ts - ) AS account_balance, + ) as account_balance, SUM( events.amount_delta ) over ( - PARTITION BY events.collateral_id - ORDER BY + partition by events.collateral_id + order by events.ts - ) AS total_balance - FROM + ) as total_balance + from ( - SELECT + select ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'transfer' AS event_type - FROM + 'transfer' as event_type + from transfers - UNION ALL - SELECT + union all + select ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'liquidation' AS event_type - FROM + 'liquidation' as event_type + from liquidations - ) events - JOIN {{ ref('arbitrum_mainnet_synths') }} - synths - ON events.collateral_id = synths.synth_market_id + ) as events + inner join + {{ ref('arbitrum_mainnet_synths') }} + as synths + on events.collateral_id = synths.synth_market_id ) -SELECT + +select ts, transaction_hash, event_type, @@ -126,5 +135,5 @@ SELECT amount_delta, account_balance, total_balance -FROM +from net_transfers diff --git a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql index cdee3a83..d1240f7e 100644 --- a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql +++ b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql @@ -1,69 +1,75 @@ -WITH synths AS ( - SELECT - synth_market_id AS collateral_id, +with synths as ( + select + synth_market_id as collateral_id, synth_token_address - FROM + from {{ ref('spot_synth_registered_arbitrum_sepolia') }} ), -transfers AS ( - SELECT + +transfers as ( + select cm.block_number, - cm.block_timestamp AS ts, + cm.block_timestamp as ts, cm.transaction_hash, cm.collateral_id, synths.synth_token_address, account_id, {{ convert_wei( 'cm.amount_delta' - ) }} AS amount_delta - FROM + ) }} as amount_delta + from {{ ref('perp_collateral_modified_arbitrum_sepolia') }} - cm - JOIN synths - ON cm.collateral_id = synths.collateral_id + as cm + inner join synths + on cm.collateral_id = synths.collateral_id ), -liq_tx AS ( - SELECT - DISTINCT transaction_hash, + +liq_tx as ( + select distinct + transaction_hash, account_id - FROM + from {{ ref('perp_account_liquidation_attempt_arbitrum_sepolia') }} ), -distributors AS ( - SELECT + +distributors as ( + select CAST( - rd.distributor_address AS text - ) AS distributor_address, + rd.distributor_address as text + ) as distributor_address, CAST( - rd.token_symbol AS text - ) AS token_symbol, + rd.token_symbol as text + ) as token_symbol, rd.synth_token_address, synths.collateral_id - FROM + from {{ ref('arbitrum_sepolia_reward_distributors') }} - rd - JOIN synths - ON synths.synth_token_address = rd.synth_token_address + as rd + inner join synths + on rd.synth_token_address = synths.synth_token_address ), -liquidations AS ( - SELECT + +liquidations as ( + select block_number, - block_timestamp AS ts,- amount / 1e18 AS amount_delta, + block_timestamp as ts, + -amount / 1e18 as amount_delta, liq_tx.transaction_hash, collateral_type, token_symbol, synth_token_address, account_id, distributors.collateral_id - FROM - prod_raw_arbitrum_sepolia.core_rewards_distributed_arbitrum_sepolia rd - JOIN liq_tx - ON rd.transaction_hash = liq_tx.transaction_hash - JOIN distributors - ON rd.distributor = distributors.distributor_address + from + prod_raw_arbitrum_sepolia.core_rewards_distributed_arbitrum_sepolia as rd + inner join liq_tx + on rd.transaction_hash = liq_tx.transaction_hash + inner join distributors + on rd.distributor = distributors.distributor_address ), -net_transfers AS ( - SELECT + +net_transfers as ( + select events.ts, events.transaction_hash, events.event_type, @@ -75,47 +81,50 @@ net_transfers AS ( SUM( events.amount_delta ) over ( - PARTITION BY events.account_id, - events.collateral_id - ORDER BY + partition by + events.account_id, + events.collateral_id + order by events.ts - ) AS account_balance, + ) as account_balance, SUM( events.amount_delta ) over ( - PARTITION BY events.collateral_id - ORDER BY + partition by events.collateral_id + order by events.ts - ) AS total_balance - FROM + ) as total_balance + from ( - SELECT + select ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'transfer' AS event_type - FROM + 'transfer' as event_type + from transfers - UNION ALL - SELECT + union all + select ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'liquidation' AS event_type - FROM + 'liquidation' as event_type + from liquidations - ) events - JOIN {{ ref('arbitrum_sepolia_synths') }} - synths - ON events.collateral_id = synths.synth_market_id + ) as events + inner join + {{ ref('arbitrum_sepolia_synths') }} + as synths + on events.collateral_id = synths.synth_market_id ) -SELECT + +select ts, transaction_hash, event_type, @@ -126,5 +135,5 @@ SELECT amount_delta, account_balance, total_balance -FROM +from net_transfers From 515771a33fe3ab3807acb0fa956432d5308e5e7d Mon Sep 17 00:00:00 2001 From: Troy Date: Tue, 5 Nov 2024 11:57:43 -0700 Subject: [PATCH 07/11] update models for prices --- ...p_collateral_balances_arbitrum_mainnet.sql | 139 +++++++++--------- .../marts/arbitrum/mainnet/perp/schema.yml | 15 ++ ...p_collateral_balances_arbitrum_sepolia.sql | 139 +++++++++--------- .../marts/arbitrum/sepolia/perp/schema.yml | 15 ++ .../seeds/synths/arbitrum_mainnet_synths.csv | 4 +- 5 files changed, 170 insertions(+), 142 deletions(-) diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql index 0573b5ef..5b95f608 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql @@ -1,75 +1,67 @@ -with synths as ( - select - synth_market_id as collateral_id, +WITH synths AS ( + SELECT + synth_market_id AS collateral_id, synth_token_address - from + FROM {{ ref('spot_synth_registered_arbitrum_mainnet') }} ), - -transfers as ( - select +transfers AS ( + SELECT cm.block_number, - cm.block_timestamp as ts, + cm.block_timestamp AS ts, cm.transaction_hash, cm.collateral_id, synths.synth_token_address, account_id, {{ convert_wei( 'cm.amount_delta' - ) }} as amount_delta - from - {{ ref('perp_collateral_modified_arbitrum_mainnet') }} - as cm - inner join synths - on cm.collateral_id = synths.collateral_id + ) }} AS amount_delta + FROM + {{ ref('perp_collateral_modified_arbitrum_mainnet') }} AS cm + INNER JOIN synths + ON cm.collateral_id = synths.collateral_id ), - -liq_tx as ( - select distinct - transaction_hash, +liq_tx AS ( + SELECT + DISTINCT transaction_hash, account_id - from + FROM {{ ref('perp_account_liquidation_attempt_arbitrum_mainnet') }} ), - -distributors as ( - select +distributors AS ( + SELECT CAST( - rd.distributor_address as text - ) as distributor_address, + rd.distributor_address AS text + ) AS distributor_address, CAST( - rd.token_symbol as text - ) as token_symbol, + rd.token_symbol AS text + ) AS token_symbol, rd.synth_token_address, synths.collateral_id - from - {{ ref('arbitrum_mainnet_reward_distributors') }} - as rd - inner join synths - on rd.synth_token_address = synths.synth_token_address + FROM + {{ ref('arbitrum_mainnet_reward_distributors') }} AS rd + INNER JOIN synths + ON rd.synth_token_address = synths.synth_token_address ), - -liquidations as ( - select +liquidations AS ( + SELECT block_number, - block_timestamp as ts, - -amount / 1e18 as amount_delta, + block_timestamp AS ts,- amount / 1e18 AS amount_delta, liq_tx.transaction_hash, collateral_type, token_symbol, synth_token_address, account_id, distributors.collateral_id - from - prod_raw_arbitrum_mainnet.core_rewards_distributed_arbitrum_mainnet as rd - inner join liq_tx - on rd.transaction_hash = liq_tx.transaction_hash - inner join distributors - on rd.distributor = distributors.distributor_address + FROM + prod_raw_arbitrum_mainnet.core_rewards_distributed_arbitrum_mainnet AS rd + INNER JOIN liq_tx + ON rd.transaction_hash = liq_tx.transaction_hash + INNER JOIN distributors + ON rd.distributor = distributors.distributor_address ), - -net_transfers as ( - select +net_transfers AS ( + SELECT events.ts, events.transaction_hash, events.event_type, @@ -77,54 +69,57 @@ net_transfers as ( events.synth_token_address, synths.synth_symbol, events.account_id, + prices.price, events.amount_delta, SUM( events.amount_delta ) over ( - partition by - events.account_id, - events.collateral_id - order by + PARTITION BY events.account_id, + events.collateral_id + ORDER BY events.ts - ) as account_balance, + ) AS account_balance, SUM( events.amount_delta ) over ( - partition by events.collateral_id - order by + PARTITION BY events.collateral_id + ORDER BY events.ts - ) as total_balance - from + ) AS total_balance + FROM ( - select + SELECT ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'transfer' as event_type - from + 'transfer' AS event_type + FROM transfers - union all - select + UNION ALL + SELECT ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'liquidation' as event_type - from + 'liquidation' AS event_type + FROM liquidations - ) as events - inner join - {{ ref('arbitrum_mainnet_synths') }} - as synths - on events.collateral_id = synths.synth_market_id + ) AS events + INNER JOIN {{ ref('arbitrum_mainnet_synths') }} AS synths + ON events.collateral_id = synths.synth_market_id + LEFT JOIN {{ ref('fct_prices_hourly_arbitrum_mainnet') }} AS prices + ON synths.token_symbol = prices.market_symbol + AND DATE_TRUNC( + 'hour', + events.ts + ) = prices.ts ) - -select +SELECT ts, transaction_hash, event_type, @@ -132,8 +127,12 @@ select synth_token_address, synth_symbol, account_id, + price, amount_delta, + amount_delta * price AS amount_delta_usd, account_balance, - total_balance -from + account_balance * price AS account_balance_usd, + total_balance, + total_balance * price AS total_balance_usd +FROM net_transfers diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml index 25aaba94..da65de85 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml @@ -715,3 +715,18 @@ models: - dbt_utils.accepted_range: min_value: 0 inclusive: true + - name: amount_delta_usd + description: "USD value of the collateral token that was transferred" + data_type: numeric + tests: + - not_null + - name: account_balance_usd + description: "USD value of the collateral token in the account after the transfer" + data_type: numeric + tests: + - not_null + - name: total_balance_usd + description: "Total USD value of the collateral token in the perps system after the transfer" + data_type: numeric + tests: + - not_null diff --git a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql index d1240f7e..8da76d69 100644 --- a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql +++ b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql @@ -1,75 +1,67 @@ -with synths as ( - select - synth_market_id as collateral_id, +WITH synths AS ( + SELECT + synth_market_id AS collateral_id, synth_token_address - from + FROM {{ ref('spot_synth_registered_arbitrum_sepolia') }} ), - -transfers as ( - select +transfers AS ( + SELECT cm.block_number, - cm.block_timestamp as ts, + cm.block_timestamp AS ts, cm.transaction_hash, cm.collateral_id, synths.synth_token_address, account_id, {{ convert_wei( 'cm.amount_delta' - ) }} as amount_delta - from - {{ ref('perp_collateral_modified_arbitrum_sepolia') }} - as cm - inner join synths - on cm.collateral_id = synths.collateral_id + ) }} AS amount_delta + FROM + {{ ref('perp_collateral_modified_arbitrum_sepolia') }} AS cm + INNER JOIN synths + ON cm.collateral_id = synths.collateral_id ), - -liq_tx as ( - select distinct - transaction_hash, +liq_tx AS ( + SELECT + DISTINCT transaction_hash, account_id - from + FROM {{ ref('perp_account_liquidation_attempt_arbitrum_sepolia') }} ), - -distributors as ( - select +distributors AS ( + SELECT CAST( - rd.distributor_address as text - ) as distributor_address, + rd.distributor_address AS text + ) AS distributor_address, CAST( - rd.token_symbol as text - ) as token_symbol, + rd.token_symbol AS text + ) AS token_symbol, rd.synth_token_address, synths.collateral_id - from - {{ ref('arbitrum_sepolia_reward_distributors') }} - as rd - inner join synths - on rd.synth_token_address = synths.synth_token_address + FROM + {{ ref('arbitrum_sepolia_reward_distributors') }} AS rd + INNER JOIN synths + ON rd.synth_token_address = synths.synth_token_address ), - -liquidations as ( - select +liquidations AS ( + SELECT block_number, - block_timestamp as ts, - -amount / 1e18 as amount_delta, + block_timestamp AS ts,- amount / 1e18 AS amount_delta, liq_tx.transaction_hash, collateral_type, token_symbol, synth_token_address, account_id, distributors.collateral_id - from - prod_raw_arbitrum_sepolia.core_rewards_distributed_arbitrum_sepolia as rd - inner join liq_tx - on rd.transaction_hash = liq_tx.transaction_hash - inner join distributors - on rd.distributor = distributors.distributor_address + FROM + prod_raw_arbitrum_sepolia.core_rewards_distributed_arbitrum_sepolia AS rd + INNER JOIN liq_tx + ON rd.transaction_hash = liq_tx.transaction_hash + INNER JOIN distributors + ON rd.distributor = distributors.distributor_address ), - -net_transfers as ( - select +net_transfers AS ( + SELECT events.ts, events.transaction_hash, events.event_type, @@ -77,54 +69,57 @@ net_transfers as ( events.synth_token_address, synths.synth_symbol, events.account_id, + prices.price, events.amount_delta, SUM( events.amount_delta ) over ( - partition by - events.account_id, - events.collateral_id - order by + PARTITION BY events.account_id, + events.collateral_id + ORDER BY events.ts - ) as account_balance, + ) AS account_balance, SUM( events.amount_delta ) over ( - partition by events.collateral_id - order by + PARTITION BY events.collateral_id + ORDER BY events.ts - ) as total_balance - from + ) AS total_balance + FROM ( - select + SELECT ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'transfer' as event_type - from + 'transfer' AS event_type + FROM transfers - union all - select + UNION ALL + SELECT ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'liquidation' as event_type - from + 'liquidation' AS event_type + FROM liquidations - ) as events - inner join - {{ ref('arbitrum_sepolia_synths') }} - as synths - on events.collateral_id = synths.synth_market_id + ) AS events + INNER JOIN {{ ref('arbitrum_sepolia_synths') }} AS synths + ON events.collateral_id = synths.synth_market_id + LEFT JOIN {{ ref('fct_prices_hourly_arbitrum_sepolia') }} AS prices + ON synths.token_symbol = prices.market_symbol + AND DATE_TRUNC( + 'hour', + events.ts + ) = prices.ts ) - -select +SELECT ts, transaction_hash, event_type, @@ -132,8 +127,12 @@ select synth_token_address, synth_symbol, account_id, + price, amount_delta, + amount_delta * price AS amount_delta_usd, account_balance, - total_balance -from + account_balance * price AS account_balance_usd, + total_balance, + total_balance * price AS total_balance_usd +FROM net_transfers diff --git a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml index e20cb81b..3b2e83ee 100644 --- a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml +++ b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/schema.yml @@ -715,3 +715,18 @@ models: - dbt_utils.accepted_range: min_value: 0 inclusive: true + - name: amount_delta_usd + description: "USD value of the collateral token that was transferred" + data_type: numeric + tests: + - not_null + - name: account_balance_usd + description: "USD value of the collateral token in the account after the transfer" + data_type: numeric + tests: + - not_null + - name: total_balance_usd + description: "Total USD value of the collateral token in the perps system after the transfer" + data_type: numeric + tests: + - not_null diff --git a/transformers/synthetix/seeds/synths/arbitrum_mainnet_synths.csv b/transformers/synthetix/seeds/synths/arbitrum_mainnet_synths.csv index 78d1eaf2..77724a7d 100644 --- a/transformers/synthetix/seeds/synths/arbitrum_mainnet_synths.csv +++ b/transformers/synthetix/seeds/synths/arbitrum_mainnet_synths.csv @@ -1,7 +1,7 @@ synth_market_id,synth_token_address,synth_name,synth_symbol,token_symbol 0,0xb2F30A7C980f052f02563fb518dcc39e6bf38175,Synthetix USD,sUSD,sUSD 2,0xE81Be4495f138FAE5846d21AC2cA822BEf452365,Synthetic USD Coin,sUSDC,USDC -3,0xFA86aB03412Ab63Fea238d43D1E839c4F7A80232,Synthetic Bitcoin,stBTC,tBTC +3,0xFA86aB03412Ab63Fea238d43D1E839c4F7A80232,Synthetic Bitcoin,stBTC,BTC 4,0x3Ec3FedA50b718b5A9ff387F93EeA7841D795B1E,Synthetic Ether,sETH,ETH 5,0xE3eE09c200584228F7C45d50E12BcC3fb65c19Ca,Synthetic USDe,sUSDe,USDe -6,0x7301a8DBd293b85A06726aE12E433a829ba3B871,Synthetic Wormhole SOL,swSOL,wSOL +6,0x7301a8DBd293b85A06726aE12E433a829ba3B871,Synthetic Wormhole SOL,swSOL,SOL From b970ca5566751bda9ab20108af169ba8c27c1050 Mon Sep 17 00:00:00 2001 From: Troy Date: Tue, 5 Nov 2024 12:13:09 -0700 Subject: [PATCH 08/11] lint --- ...p_collateral_balances_arbitrum_mainnet.sql | 141 ++++++++++-------- ...p_collateral_balances_arbitrum_sepolia.sql | 141 ++++++++++-------- 2 files changed, 150 insertions(+), 132 deletions(-) diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql index 5b95f608..5b3273c5 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql @@ -1,67 +1,73 @@ -WITH synths AS ( - SELECT - synth_market_id AS collateral_id, +with synths as ( + select + synth_market_id as collateral_id, synth_token_address - FROM + from {{ ref('spot_synth_registered_arbitrum_mainnet') }} ), -transfers AS ( - SELECT + +transfers as ( + select cm.block_number, - cm.block_timestamp AS ts, + cm.block_timestamp as ts, cm.transaction_hash, cm.collateral_id, synths.synth_token_address, account_id, {{ convert_wei( 'cm.amount_delta' - ) }} AS amount_delta - FROM - {{ ref('perp_collateral_modified_arbitrum_mainnet') }} AS cm - INNER JOIN synths - ON cm.collateral_id = synths.collateral_id + ) }} as amount_delta + from + {{ ref('perp_collateral_modified_arbitrum_mainnet') }} as cm + inner join synths + on cm.collateral_id = synths.collateral_id ), -liq_tx AS ( - SELECT - DISTINCT transaction_hash, + +liq_tx as ( + select distinct + transaction_hash, account_id - FROM + from {{ ref('perp_account_liquidation_attempt_arbitrum_mainnet') }} ), -distributors AS ( - SELECT + +distributors as ( + select CAST( - rd.distributor_address AS text - ) AS distributor_address, + rd.distributor_address as text + ) as distributor_address, CAST( - rd.token_symbol AS text - ) AS token_symbol, + rd.token_symbol as text + ) as token_symbol, rd.synth_token_address, synths.collateral_id - FROM - {{ ref('arbitrum_mainnet_reward_distributors') }} AS rd - INNER JOIN synths - ON rd.synth_token_address = synths.synth_token_address + from + {{ ref('arbitrum_mainnet_reward_distributors') }} as rd + inner join synths + on rd.synth_token_address = synths.synth_token_address ), -liquidations AS ( - SELECT + +liquidations as ( + select block_number, - block_timestamp AS ts,- amount / 1e18 AS amount_delta, + block_timestamp as ts, + -amount / 1e18 as amount_delta, liq_tx.transaction_hash, collateral_type, token_symbol, synth_token_address, account_id, distributors.collateral_id - FROM - prod_raw_arbitrum_mainnet.core_rewards_distributed_arbitrum_mainnet AS rd - INNER JOIN liq_tx - ON rd.transaction_hash = liq_tx.transaction_hash - INNER JOIN distributors - ON rd.distributor = distributors.distributor_address + from + prod_raw_arbitrum_mainnet.core_rewards_distributed_arbitrum_mainnet as rd + inner join liq_tx + on rd.transaction_hash = liq_tx.transaction_hash + inner join distributors + on rd.distributor = distributors.distributor_address ), -net_transfers AS ( - SELECT + +net_transfers as ( + select events.ts, events.transaction_hash, events.event_type, @@ -74,52 +80,55 @@ net_transfers AS ( SUM( events.amount_delta ) over ( - PARTITION BY events.account_id, - events.collateral_id - ORDER BY + partition by + events.account_id, + events.collateral_id + order by events.ts - ) AS account_balance, + ) as account_balance, SUM( events.amount_delta ) over ( - PARTITION BY events.collateral_id - ORDER BY + partition by events.collateral_id + order by events.ts - ) AS total_balance - FROM + ) as total_balance + from ( - SELECT + select ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'transfer' AS event_type - FROM + 'transfer' as event_type + from transfers - UNION ALL - SELECT + union all + select ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'liquidation' AS event_type - FROM + 'liquidation' as event_type + from liquidations - ) AS events - INNER JOIN {{ ref('arbitrum_mainnet_synths') }} AS synths - ON events.collateral_id = synths.synth_market_id - LEFT JOIN {{ ref('fct_prices_hourly_arbitrum_mainnet') }} AS prices - ON synths.token_symbol = prices.market_symbol - AND DATE_TRUNC( - 'hour', - events.ts - ) = prices.ts + ) as events + inner join {{ ref('arbitrum_mainnet_synths') }} as synths + on events.collateral_id = synths.synth_market_id + left join {{ ref('fct_prices_hourly_arbitrum_mainnet') }} as prices + on + synths.token_symbol = prices.market_symbol + and DATE_TRUNC( + 'hour', + events.ts + ) = prices.ts ) -SELECT + +select ts, transaction_hash, event_type, @@ -129,10 +138,10 @@ SELECT account_id, price, amount_delta, - amount_delta * price AS amount_delta_usd, + amount_delta * price as amount_delta_usd, account_balance, - account_balance * price AS account_balance_usd, + account_balance * price as account_balance_usd, total_balance, - total_balance * price AS total_balance_usd -FROM + total_balance * price as total_balance_usd +from net_transfers diff --git a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql index 8da76d69..aa5a0e5c 100644 --- a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql +++ b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql @@ -1,67 +1,73 @@ -WITH synths AS ( - SELECT - synth_market_id AS collateral_id, +with synths as ( + select + synth_market_id as collateral_id, synth_token_address - FROM + from {{ ref('spot_synth_registered_arbitrum_sepolia') }} ), -transfers AS ( - SELECT + +transfers as ( + select cm.block_number, - cm.block_timestamp AS ts, + cm.block_timestamp as ts, cm.transaction_hash, cm.collateral_id, synths.synth_token_address, account_id, {{ convert_wei( 'cm.amount_delta' - ) }} AS amount_delta - FROM - {{ ref('perp_collateral_modified_arbitrum_sepolia') }} AS cm - INNER JOIN synths - ON cm.collateral_id = synths.collateral_id + ) }} as amount_delta + from + {{ ref('perp_collateral_modified_arbitrum_sepolia') }} as cm + inner join synths + on cm.collateral_id = synths.collateral_id ), -liq_tx AS ( - SELECT - DISTINCT transaction_hash, + +liq_tx as ( + select distinct + transaction_hash, account_id - FROM + from {{ ref('perp_account_liquidation_attempt_arbitrum_sepolia') }} ), -distributors AS ( - SELECT + +distributors as ( + select CAST( - rd.distributor_address AS text - ) AS distributor_address, + rd.distributor_address as text + ) as distributor_address, CAST( - rd.token_symbol AS text - ) AS token_symbol, + rd.token_symbol as text + ) as token_symbol, rd.synth_token_address, synths.collateral_id - FROM - {{ ref('arbitrum_sepolia_reward_distributors') }} AS rd - INNER JOIN synths - ON rd.synth_token_address = synths.synth_token_address + from + {{ ref('arbitrum_sepolia_reward_distributors') }} as rd + inner join synths + on rd.synth_token_address = synths.synth_token_address ), -liquidations AS ( - SELECT + +liquidations as ( + select block_number, - block_timestamp AS ts,- amount / 1e18 AS amount_delta, + block_timestamp as ts, + -amount / 1e18 as amount_delta, liq_tx.transaction_hash, collateral_type, token_symbol, synth_token_address, account_id, distributors.collateral_id - FROM - prod_raw_arbitrum_sepolia.core_rewards_distributed_arbitrum_sepolia AS rd - INNER JOIN liq_tx - ON rd.transaction_hash = liq_tx.transaction_hash - INNER JOIN distributors - ON rd.distributor = distributors.distributor_address + from + prod_raw_arbitrum_sepolia.core_rewards_distributed_arbitrum_sepolia as rd + inner join liq_tx + on rd.transaction_hash = liq_tx.transaction_hash + inner join distributors + on rd.distributor = distributors.distributor_address ), -net_transfers AS ( - SELECT + +net_transfers as ( + select events.ts, events.transaction_hash, events.event_type, @@ -74,52 +80,55 @@ net_transfers AS ( SUM( events.amount_delta ) over ( - PARTITION BY events.account_id, - events.collateral_id - ORDER BY + partition by + events.account_id, + events.collateral_id + order by events.ts - ) AS account_balance, + ) as account_balance, SUM( events.amount_delta ) over ( - PARTITION BY events.collateral_id - ORDER BY + partition by events.collateral_id + order by events.ts - ) AS total_balance - FROM + ) as total_balance + from ( - SELECT + select ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'transfer' AS event_type - FROM + 'transfer' as event_type + from transfers - UNION ALL - SELECT + union all + select ts, transaction_hash, collateral_id, synth_token_address, account_id, amount_delta, - 'liquidation' AS event_type - FROM + 'liquidation' as event_type + from liquidations - ) AS events - INNER JOIN {{ ref('arbitrum_sepolia_synths') }} AS synths - ON events.collateral_id = synths.synth_market_id - LEFT JOIN {{ ref('fct_prices_hourly_arbitrum_sepolia') }} AS prices - ON synths.token_symbol = prices.market_symbol - AND DATE_TRUNC( - 'hour', - events.ts - ) = prices.ts + ) as events + inner join {{ ref('arbitrum_sepolia_synths') }} as synths + on events.collateral_id = synths.synth_market_id + left join {{ ref('fct_prices_hourly_arbitrum_sepolia') }} as prices + on + synths.token_symbol = prices.market_symbol + and DATE_TRUNC( + 'hour', + events.ts + ) = prices.ts ) -SELECT + +select ts, transaction_hash, event_type, @@ -129,10 +138,10 @@ SELECT account_id, price, amount_delta, - amount_delta * price AS amount_delta_usd, + amount_delta * price as amount_delta_usd, account_balance, - account_balance * price AS account_balance_usd, + account_balance * price as account_balance_usd, total_balance, - total_balance * price AS total_balance_usd -FROM + total_balance * price as total_balance_usd +from net_transfers From 3dcf8ce189b89d7361404168c1f93e06036365db Mon Sep 17 00:00:00 2001 From: Troy Date: Tue, 5 Nov 2024 13:38:58 -0700 Subject: [PATCH 09/11] fix linting errors --- ...p_collateral_balances_arbitrum_mainnet.sql | 128 ++++++++---------- ...p_collateral_balances_arbitrum_sepolia.sql | 128 ++++++++---------- 2 files changed, 110 insertions(+), 146 deletions(-) diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql index 5b3273c5..6a049307 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql @@ -13,10 +13,8 @@ transfers as ( cm.transaction_hash, cm.collateral_id, synths.synth_token_address, - account_id, - {{ convert_wei( - 'cm.amount_delta' - ) }} as amount_delta + cm.account_id, + {{ convert_wei('cm.amount_delta') }} as amount_delta from {{ ref('perp_collateral_modified_arbitrum_mainnet') }} as cm inner join synths @@ -33,12 +31,8 @@ liq_tx as ( distributors as ( select - CAST( - rd.distributor_address as text - ) as distributor_address, - CAST( - rd.token_symbol as text - ) as token_symbol, + CAST(rd.distributor_address as text) as distributor_address, + CAST(rd.token_symbol as text) as token_symbol, rd.synth_token_address, synths.collateral_id from @@ -49,17 +43,17 @@ distributors as ( liquidations as ( select - block_number, - block_timestamp as ts, - -amount / 1e18 as amount_delta, + rd.block_number, + rd.block_timestamp as ts, + -rd.amount / 1e18 as amount_delta, liq_tx.transaction_hash, - collateral_type, - token_symbol, - synth_token_address, - account_id, + rd.collateral_type, + distributors.token_symbol, + distributors.synth_token_address, + liq_tx.account_id, distributors.collateral_id from - prod_raw_arbitrum_mainnet.core_rewards_distributed_arbitrum_mainnet as rd + {{ ref('core_rewards_distributed_arbitrum_mainnet') }} as rd inner join liq_tx on rd.transaction_hash = liq_tx.transaction_hash inner join distributors @@ -77,71 +71,59 @@ net_transfers as ( events.account_id, prices.price, events.amount_delta, - SUM( - events.amount_delta - ) over ( - partition by - events.account_id, - events.collateral_id - order by - events.ts + SUM(events.amount_delta) over ( + partition by events.account_id, events.collateral_id + order by events.ts ) as account_balance, - SUM( - events.amount_delta - ) over ( + SUM(events.amount_delta) over ( partition by events.collateral_id - order by - events.ts + order by events.ts ) as total_balance - from - ( - select - ts, - transaction_hash, - collateral_id, - synth_token_address, - account_id, - amount_delta, - 'transfer' as event_type - from - transfers - union all - select - ts, - transaction_hash, - collateral_id, - synth_token_address, - account_id, - amount_delta, - 'liquidation' as event_type - from - liquidations - ) as events + from ( + select + transfers.ts, + transfers.transaction_hash, + transfers.collateral_id, + transfers.synth_token_address, + transfers.account_id, + transfers.amount_delta, + 'transfer' as event_type + from + transfers + union all + select + liquidations.ts, + liquidations.transaction_hash, + liquidations.collateral_id, + liquidations.synth_token_address, + liquidations.account_id, + liquidations.amount_delta, + 'liquidation' as event_type + from + liquidations + ) as events inner join {{ ref('arbitrum_mainnet_synths') }} as synths on events.collateral_id = synths.synth_market_id left join {{ ref('fct_prices_hourly_arbitrum_mainnet') }} as prices on synths.token_symbol = prices.market_symbol - and DATE_TRUNC( - 'hour', - events.ts - ) = prices.ts + and DATE_TRUNC('hour', events.ts) = prices.ts ) select - ts, - transaction_hash, - event_type, - collateral_id, - synth_token_address, - synth_symbol, - account_id, - price, - amount_delta, - amount_delta * price as amount_delta_usd, - account_balance, - account_balance * price as account_balance_usd, - total_balance, - total_balance * price as total_balance_usd + net_transfers.ts, + net_transfers.transaction_hash, + net_transfers.event_type, + net_transfers.collateral_id, + net_transfers.synth_token_address, + net_transfers.synth_symbol, + net_transfers.account_id, + net_transfers.price, + net_transfers.amount_delta, + net_transfers.amount_delta * net_transfers.price as amount_delta_usd, + net_transfers.account_balance, + net_transfers.account_balance * net_transfers.price as account_balance_usd, + net_transfers.total_balance, + net_transfers.total_balance * net_transfers.price as total_balance_usd from net_transfers diff --git a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql index aa5a0e5c..b0633d93 100644 --- a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql +++ b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql @@ -13,10 +13,8 @@ transfers as ( cm.transaction_hash, cm.collateral_id, synths.synth_token_address, - account_id, - {{ convert_wei( - 'cm.amount_delta' - ) }} as amount_delta + cm.account_id, + {{ convert_wei('cm.amount_delta') }} as amount_delta from {{ ref('perp_collateral_modified_arbitrum_sepolia') }} as cm inner join synths @@ -33,12 +31,8 @@ liq_tx as ( distributors as ( select - CAST( - rd.distributor_address as text - ) as distributor_address, - CAST( - rd.token_symbol as text - ) as token_symbol, + CAST(rd.distributor_address as text) as distributor_address, + CAST(rd.token_symbol as text) as token_symbol, rd.synth_token_address, synths.collateral_id from @@ -49,17 +43,17 @@ distributors as ( liquidations as ( select - block_number, - block_timestamp as ts, - -amount / 1e18 as amount_delta, + rd.block_number, + rd.block_timestamp as ts, + -rd.amount / 1e18 as amount_delta, liq_tx.transaction_hash, - collateral_type, - token_symbol, - synth_token_address, - account_id, + rd.collateral_type, + distributors.token_symbol, + distributors.synth_token_address, + liq_tx.account_id, distributors.collateral_id from - prod_raw_arbitrum_sepolia.core_rewards_distributed_arbitrum_sepolia as rd + {{ ref('core_rewards_distributed_arbitrum_sepolia') }} as rd inner join liq_tx on rd.transaction_hash = liq_tx.transaction_hash inner join distributors @@ -77,71 +71,59 @@ net_transfers as ( events.account_id, prices.price, events.amount_delta, - SUM( - events.amount_delta - ) over ( - partition by - events.account_id, - events.collateral_id - order by - events.ts + SUM(events.amount_delta) over ( + partition by events.account_id, events.collateral_id + order by events.ts ) as account_balance, - SUM( - events.amount_delta - ) over ( + SUM(events.amount_delta) over ( partition by events.collateral_id - order by - events.ts + order by events.ts ) as total_balance - from - ( - select - ts, - transaction_hash, - collateral_id, - synth_token_address, - account_id, - amount_delta, - 'transfer' as event_type - from - transfers - union all - select - ts, - transaction_hash, - collateral_id, - synth_token_address, - account_id, - amount_delta, - 'liquidation' as event_type - from - liquidations - ) as events + from ( + select + transfers.ts, + transfers.transaction_hash, + transfers.collateral_id, + transfers.synth_token_address, + transfers.account_id, + transfers.amount_delta, + 'transfer' as event_type + from + transfers + union all + select + liquidations.ts, + liquidations.transaction_hash, + liquidations.collateral_id, + liquidations.synth_token_address, + liquidations.account_id, + liquidations.amount_delta, + 'liquidation' as event_type + from + liquidations + ) as events inner join {{ ref('arbitrum_sepolia_synths') }} as synths on events.collateral_id = synths.synth_market_id left join {{ ref('fct_prices_hourly_arbitrum_sepolia') }} as prices on synths.token_symbol = prices.market_symbol - and DATE_TRUNC( - 'hour', - events.ts - ) = prices.ts + and DATE_TRUNC('hour', events.ts) = prices.ts ) select - ts, - transaction_hash, - event_type, - collateral_id, - synth_token_address, - synth_symbol, - account_id, - price, - amount_delta, - amount_delta * price as amount_delta_usd, - account_balance, - account_balance * price as account_balance_usd, - total_balance, - total_balance * price as total_balance_usd + net_transfers.ts, + net_transfers.transaction_hash, + net_transfers.event_type, + net_transfers.collateral_id, + net_transfers.synth_token_address, + net_transfers.synth_symbol, + net_transfers.account_id, + net_transfers.price, + net_transfers.amount_delta, + net_transfers.amount_delta * net_transfers.price as amount_delta_usd, + net_transfers.account_balance, + net_transfers.account_balance * net_transfers.price as account_balance_usd, + net_transfers.total_balance, + net_transfers.total_balance * net_transfers.price as total_balance_usd from net_transfers From 38f4b555b18f87868a2e01f6f542b2b857688a65 Mon Sep 17 00:00:00 2001 From: Troy Date: Tue, 5 Nov 2024 14:26:50 -0700 Subject: [PATCH 10/11] fix account_id type --- ...p_collateral_balances_arbitrum_mainnet.sql | 6 ++--- ...p_collateral_modified_arbitrum_mainnet.sql | 25 +++++++++++-------- ...p_collateral_balances_arbitrum_sepolia.sql | 6 ++--- ...p_collateral_modified_arbitrum_sepolia.sql | 25 +++++++++++-------- 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql index 6a049307..3386949c 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql @@ -13,7 +13,7 @@ transfers as ( cm.transaction_hash, cm.collateral_id, synths.synth_token_address, - cm.account_id, + CAST(cm.account_id as text) as account_id, {{ convert_wei('cm.amount_delta') }} as amount_delta from {{ ref('perp_collateral_modified_arbitrum_mainnet') }} as cm @@ -24,7 +24,7 @@ transfers as ( liq_tx as ( select distinct transaction_hash, - account_id + CAST(account_id as text) as account_id from {{ ref('perp_account_liquidation_attempt_arbitrum_mainnet') }} ), @@ -50,7 +50,7 @@ liquidations as ( rd.collateral_type, distributors.token_symbol, distributors.synth_token_address, - liq_tx.account_id, + CAST(liq_tx.account_id as text) as account_id, distributors.collateral_id from {{ ref('core_rewards_distributed_arbitrum_mainnet') }} as rd diff --git a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_modified_arbitrum_mainnet.sql b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_modified_arbitrum_mainnet.sql index 53f594a5..686154ca 100644 --- a/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_modified_arbitrum_mainnet.sql +++ b/transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_modified_arbitrum_mainnet.sql @@ -1,13 +1,18 @@ select - id, - block_timestamp, - account_id, - block_number, - transaction_hash, - contract, - event_name, - collateral_id as synth_market_id, - sender, - {{ convert_wei("amount_delta") }} as amount_delta + cm.id, + cm.block_timestamp, + cm.account_id, + cm.block_number, + cm.transaction_hash, + cm.contract, + cm.event_name, + synths.synth_symbol, + cm.collateral_id as synth_market_id, + synths.synth_token_address, + cm.sender, + {{ convert_wei("cm.amount_delta") }} as amount_delta from {{ ref("perp_collateral_modified_arbitrum_mainnet") }} + as cm +inner join {{ ref('arbitrum_mainnet_synths') }} as synths + on cm.collateral_id = synths.synth_market_id diff --git a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql index b0633d93..2cfca30c 100644 --- a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql +++ b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql @@ -13,7 +13,7 @@ transfers as ( cm.transaction_hash, cm.collateral_id, synths.synth_token_address, - cm.account_id, + CAST(cm.account_id as text) as account_id, {{ convert_wei('cm.amount_delta') }} as amount_delta from {{ ref('perp_collateral_modified_arbitrum_sepolia') }} as cm @@ -24,7 +24,7 @@ transfers as ( liq_tx as ( select distinct transaction_hash, - account_id + CAST(account_id as text) as account_id from {{ ref('perp_account_liquidation_attempt_arbitrum_sepolia') }} ), @@ -50,7 +50,7 @@ liquidations as ( rd.collateral_type, distributors.token_symbol, distributors.synth_token_address, - liq_tx.account_id, + CAST(liq_tx.account_id as text) as account_id, distributors.collateral_id from {{ ref('core_rewards_distributed_arbitrum_sepolia') }} as rd diff --git a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_modified_arbitrum_sepolia.sql b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_modified_arbitrum_sepolia.sql index 7aa2b683..997db7a0 100644 --- a/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_modified_arbitrum_sepolia.sql +++ b/transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_modified_arbitrum_sepolia.sql @@ -1,13 +1,18 @@ select - id, - block_timestamp, - account_id, - block_number, - transaction_hash, - contract, - event_name, - collateral_id as synth_market_id, - sender, - {{ convert_wei("amount_delta") }} as amount_delta + cm.id, + cm.block_timestamp, + cm.account_id, + cm.block_number, + cm.transaction_hash, + cm.contract, + cm.event_name, + synths.synth_symbol, + cm.collateral_id as synth_market_id, + synths.synth_token_address, + cm.sender, + {{ convert_wei("cm.amount_delta") }} as amount_delta from {{ ref("perp_collateral_modified_arbitrum_sepolia") }} + as cm +inner join {{ ref('arbitrum_sepolia_synths') }} as synths + on cm.collateral_id = synths.synth_market_id From 08ff3006a86e419b5c4556249f767cca5df759df Mon Sep 17 00:00:00 2001 From: Troy Date: Tue, 5 Nov 2024 14:35:11 -0700 Subject: [PATCH 11/11] update base models --- ..._perp_collateral_modified_base_mainnet.sql | 25 +++++++++++-------- ..._perp_collateral_modified_base_sepolia.sql | 25 +++++++++++-------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/transformers/synthetix/models/marts/base/mainnet/perp/fct_perp_collateral_modified_base_mainnet.sql b/transformers/synthetix/models/marts/base/mainnet/perp/fct_perp_collateral_modified_base_mainnet.sql index 056b4c9a..05ac42e1 100644 --- a/transformers/synthetix/models/marts/base/mainnet/perp/fct_perp_collateral_modified_base_mainnet.sql +++ b/transformers/synthetix/models/marts/base/mainnet/perp/fct_perp_collateral_modified_base_mainnet.sql @@ -1,13 +1,18 @@ select - id, - block_timestamp, - account_id, - block_number, - transaction_hash, - contract, - event_name, - synth_market_id, - sender, - {{ convert_wei("amount_delta") }} as amount_delta + cm.id, + cm.block_timestamp, + cm.account_id, + cm.block_number, + cm.transaction_hash, + cm.contract, + cm.event_name, + synths.synth_symbol, + cm.collateral_id as synth_market_id, + synths.synth_token_address, + cm.sender, + {{ convert_wei("cm.amount_delta") }} as amount_delta from {{ ref("perp_collateral_modified_base_mainnet") }} + as cm +inner join {{ ref('base_mainnet_synths') }} as synths + on cm.collateral_id = synths.synth_market_id diff --git a/transformers/synthetix/models/marts/base/sepolia/perp/fct_perp_collateral_modified_base_sepolia.sql b/transformers/synthetix/models/marts/base/sepolia/perp/fct_perp_collateral_modified_base_sepolia.sql index 807167d1..629183e8 100644 --- a/transformers/synthetix/models/marts/base/sepolia/perp/fct_perp_collateral_modified_base_sepolia.sql +++ b/transformers/synthetix/models/marts/base/sepolia/perp/fct_perp_collateral_modified_base_sepolia.sql @@ -1,13 +1,18 @@ select - id, - block_timestamp, - account_id, - block_number, - transaction_hash, - contract, - event_name, - synth_market_id, - sender, - {{ convert_wei("amount_delta") }} as amount_delta + cm.id, + cm.block_timestamp, + cm.account_id, + cm.block_number, + cm.transaction_hash, + cm.contract, + cm.event_name, + synths.synth_symbol, + cm.collateral_id as synth_market_id, + synths.synth_token_address, + cm.sender, + {{ convert_wei("cm.amount_delta") }} as amount_delta from {{ ref("perp_collateral_modified_base_sepolia") }} + as cm +inner join {{ ref('base_sepolia_synths') }} as synths + on cm.collateral_id = synths.synth_market_id