Collateral balances

Makefile

@@ -24,6 +24,9 @@ extract:
 	docker compose run extractors python main.py configs/arbitrum_mainnet.yaml
 	docker compose run extractors python main.py configs/arbitrum_sepolia.yaml
 
+synths:
+	docker compose run transformer python scripts/get_synths.py
+
 wrap:
 	docker compose run transformer python scripts/wrap_tables.py
 

docker-compose.yml

@@ -170,6 +170,8 @@ services:
       context: ./transformers
     depends_on:
       - db
+    env_file:
+      - .env
     environment:
       PG_PASSWORD: $PG_PASSWORD
     volumes:

transformers/requirements.txt

@@ -3,3 +3,4 @@ pyarrow
 psycopg2-binary
 dbt-postgres
 sqlfluff
+synthetix==0.1.21

transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_balances_arbitrum_mainnet.sql

@@ -0,0 +1,129 @@
+with synths as (
+    select
+        synth_market_id as collateral_id,
+        synth_token_address
+    from
+        {{ ref('spot_synth_registered_arbitrum_mainnet') }}
+),
+
+transfers as (
+    select
+        cm.block_number,
+        cm.block_timestamp as ts,
+        cm.transaction_hash,
+        cm.collateral_id,
+        synths.synth_token_address,
+        CAST(cm.account_id as text) as account_id,
+        {{ convert_wei('cm.amount_delta') }} as amount_delta
+    from
+        {{ ref('perp_collateral_modified_arbitrum_mainnet') }} as cm
+    inner join synths
+        on cm.collateral_id = synths.collateral_id
+),
+
+liq_tx as (
+    select distinct
+        transaction_hash,
+        CAST(account_id as text) as account_id
+    from
+        {{ ref('perp_account_liquidation_attempt_arbitrum_mainnet') }}
+),
+
+distributors as (
+    select
+        CAST(rd.distributor_address as text) as distributor_address,
+        CAST(rd.token_symbol as text) as token_symbol,
+        rd.synth_token_address,
+        synths.collateral_id
+    from
+        {{ ref('arbitrum_mainnet_reward_distributors') }} as rd
+    inner join synths
+        on rd.synth_token_address = synths.synth_token_address
+),
+
+liquidations as (
+    select
+        rd.block_number,
+        rd.block_timestamp as ts,
+        -rd.amount / 1e18 as amount_delta,
+        liq_tx.transaction_hash,
+        rd.collateral_type,
+        distributors.token_symbol,
+        distributors.synth_token_address,
+        CAST(liq_tx.account_id as text) as account_id,
+        distributors.collateral_id
+    from
+        {{ ref('core_rewards_distributed_arbitrum_mainnet') }} as rd
+    inner join liq_tx
+        on rd.transaction_hash = liq_tx.transaction_hash
+    inner join distributors
+        on rd.distributor = distributors.distributor_address
+),
+
+net_transfers as (
+    select
+        events.ts,
+        events.transaction_hash,
+        events.event_type,
+        events.collateral_id,
+        events.synth_token_address,
+        synths.synth_symbol,
+        events.account_id,
+        prices.price,
+        events.amount_delta,
+        SUM(events.amount_delta) over (
+            partition by events.account_id, events.collateral_id
+            order by events.ts
+        ) as account_balance,
+        SUM(events.amount_delta) over (
+            partition by events.collateral_id
+            order by events.ts
+        ) as total_balance
+    from (
+        select
+            transfers.ts,
+            transfers.transaction_hash,
+            transfers.collateral_id,
+            transfers.synth_token_address,
+            transfers.account_id,
+            transfers.amount_delta,
+            'transfer' as event_type
+        from
+            transfers
+        union all
+        select
+            liquidations.ts,
+            liquidations.transaction_hash,
+            liquidations.collateral_id,
+            liquidations.synth_token_address,
+            liquidations.account_id,
+            liquidations.amount_delta,
+            'liquidation' as event_type
+        from
+            liquidations
+    ) as events
+    inner join {{ ref('arbitrum_mainnet_synths') }} as synths
+        on events.collateral_id = synths.synth_market_id
+    left join {{ ref('fct_prices_hourly_arbitrum_mainnet') }} as prices
+        on
+            synths.token_symbol = prices.market_symbol
+            and DATE_TRUNC('hour', events.ts) = prices.ts
+)
+
+select
+    net_transfers.ts,
+    net_transfers.transaction_hash,
+    net_transfers.event_type,
+    net_transfers.collateral_id,
+    net_transfers.synth_token_address,
+    net_transfers.synth_symbol,
+    net_transfers.account_id,
+    net_transfers.price,
+    net_transfers.amount_delta,
+    net_transfers.amount_delta * net_transfers.price as amount_delta_usd,
+    net_transfers.account_balance,
+    net_transfers.account_balance * net_transfers.price as account_balance_usd,
+    net_transfers.total_balance,
+    net_transfers.total_balance * net_transfers.price as total_balance_usd
+from
+    net_transfers

transformers/synthetix/models/marts/arbitrum/mainnet/perp/fct_perp_collateral_modified_arbitrum_mainnet.sql

@@ -1,13 +1,18 @@
 select
-    id,
-    block_timestamp,
-    account_id,
-    block_number,
-    transaction_hash,
-    contract,
-    event_name,
-    collateral_id as synth_market_id,
-    sender,
-    {{ convert_wei("amount_delta") }} as amount_delta
+    cm.id,
+    cm.block_timestamp,
+    cm.account_id,
+    cm.block_number,
+    cm.transaction_hash,
+    cm.contract,
+    cm.event_name,
+    synths.synth_symbol,
+    cm.collateral_id as synth_market_id,
+    synths.synth_token_address,
+    cm.sender,
+    {{ convert_wei("cm.amount_delta") }} as amount_delta
 from
     {{ ref("perp_collateral_modified_arbitrum_mainnet") }}
+    as cm
+inner join {{ ref('arbitrum_mainnet_synths') }} as synths
+    on cm.collateral_id = synths.synth_market_id

transformers/synthetix/models/marts/arbitrum/mainnet/perp/schema.yml

@@ -662,3 +662,71 @@ models:
         data_type: text
         tests:
           - not_null
+  - name: fct_perp_collateral_balances_arbitrum_mainnet
+    columns:
+      - name: ts
+        description: "Block timestamp"
+        data_type: timestamp with time zone
+        tests:
+          - not_null
+      - name: transaction_hash
+        description: "Transaction hash"
+        data_type: text
+        tests:
+          - not_null
+      - name: collateral_id
+        description: "Market ID of the collateral, from the spot market"
+        data_type: integer
+        tests:
+          - not_null
+      - name: synth_token_address
+        description: "Address of the synth token"
+        data_type: text
+        tests:
+          - not_null
+      - name: synth_symbol
+        description: "Symbol of the synth token"
+        data_type: text
+        tests:
+          - not_null
+      - name: event_type
+        description: "Description of the event, e.g. 'transfer' or 'liquidation'"
+        data_type: text
+        tests:
+          - not_null
+      - name: amount_delta
+        description: "Amount of the collateral token that was transferred"
+        data_type: numeric
+        tests:
+          - not_null
+      - name: account_balance
+        description: "Balance of the collateral token in the account after the transfer"
+        data_type: numeric
+        tests:
+          - not_null
+          - dbt_utils.accepted_range:
+              min_value: 0
+              inclusive: true
+      - name: total_balance
+        description: "Total balance of the collateral token in the perps system after the transfer"
+        data_type: numeric
+        tests:
+          - not_null
+          - dbt_utils.accepted_range:
+              min_value: 0
+              inclusive: true
+      - name: amount_delta_usd
+        description: "USD value of the collateral token that was transferred"
+        data_type: numeric
+        tests:
+          - not_null
+      - name: account_balance_usd
+        description: "USD value of the collateral token in the account after the transfer"
+        data_type: numeric
+        tests:
+          - not_null
+      - name: total_balance_usd
+        description: "Total USD value of the collateral token in the perps system after the transfer"
+        data_type: numeric
+        tests:
+          - not_null

transformers/synthetix/models/marts/arbitrum/sepolia/perp/fct_perp_collateral_balances_arbitrum_sepolia.sql

@@ -0,0 +1,129 @@
+with synths as (
+    select
+        synth_market_id as collateral_id,
+        synth_token_address
+    from
+        {{ ref('spot_synth_registered_arbitrum_sepolia') }}
+),
+
+transfers as (
+    select
+        cm.block_number,
+        cm.block_timestamp as ts,
+        cm.transaction_hash,
+        cm.collateral_id,
+        synths.synth_token_address,
+        CAST(cm.account_id as text) as account_id,
+        {{ convert_wei('cm.amount_delta') }} as amount_delta
+    from
+        {{ ref('perp_collateral_modified_arbitrum_sepolia') }} as cm
+    inner join synths
+        on cm.collateral_id = synths.collateral_id
+),
+
+liq_tx as (
+    select distinct
+        transaction_hash,
+        CAST(account_id as text) as account_id
+    from
+        {{ ref('perp_account_liquidation_attempt_arbitrum_sepolia') }}
+),
+
+distributors as (
+    select
+        CAST(rd.distributor_address as text) as distributor_address,
+        CAST(rd.token_symbol as text) as token_symbol,
+        rd.synth_token_address,
+        synths.collateral_id
+    from
+        {{ ref('arbitrum_sepolia_reward_distributors') }} as rd
+    inner join synths
+        on rd.synth_token_address = synths.synth_token_address
+),
+
+liquidations as (
+    select
+        rd.block_number,
+        rd.block_timestamp as ts,
+        -rd.amount / 1e18 as amount_delta,
+        liq_tx.transaction_hash,
+        rd.collateral_type,
+        distributors.token_symbol,
+        distributors.synth_token_address,
+        CAST(liq_tx.account_id as text) as account_id,
+        distributors.collateral_id
+    from
+        {{ ref('core_rewards_distributed_arbitrum_sepolia') }} as rd
+    inner join liq_tx
+        on rd.transaction_hash = liq_tx.transaction_hash
+    inner join distributors
+        on rd.distributor = distributors.distributor_address
+),
+
+net_transfers as (
+    select
+        events.ts,
+        events.transaction_hash,
+        events.event_type,
+        events.collateral_id,
+        events.synth_token_address,
+        synths.synth_symbol,
+        events.account_id,
+        prices.price,
+        events.amount_delta,
+        SUM(events.amount_delta) over (
+            partition by events.account_id, events.collateral_id
+            order by events.ts
+        ) as account_balance,
+        SUM(events.amount_delta) over (
+            partition by events.collateral_id
+            order by events.ts
+        ) as total_balance
+    from (
+        select
+            transfers.ts,
+            transfers.transaction_hash,
+            transfers.collateral_id,
+            transfers.synth_token_address,
+            transfers.account_id,
+            transfers.amount_delta,
+            'transfer' as event_type
+        from
+            transfers
+        union all
+        select
+            liquidations.ts,
+            liquidations.transaction_hash,
+            liquidations.collateral_id,
+            liquidations.synth_token_address,
+            liquidations.account_id,
+            liquidations.amount_delta,
+            'liquidation' as event_type
+        from
+            liquidations
+    ) as events
+    inner join {{ ref('arbitrum_sepolia_synths') }} as synths
+        on events.collateral_id = synths.synth_market_id
+    left join {{ ref('fct_prices_hourly_arbitrum_sepolia') }} as prices
+        on
+            synths.token_symbol = prices.market_symbol
+            and DATE_TRUNC('hour', events.ts) = prices.ts
+)
+
+select
+    net_transfers.ts,
+    net_transfers.transaction_hash,
+    net_transfers.event_type,
+    net_transfers.collateral_id,
+    net_transfers.synth_token_address,
+    net_transfers.synth_symbol,
+    net_transfers.account_id,
+    net_transfers.price,
+    net_transfers.amount_delta,
+    net_transfers.amount_delta * net_transfers.price as amount_delta_usd,
+    net_transfers.account_balance,
+    net_transfers.account_balance * net_transfers.price as account_balance_usd,
+    net_transfers.total_balance,
+    net_transfers.total_balance * net_transfers.price as total_balance_usd
+from
+    net_transfers