Skip to content

Commit

Permalink
Add support for VerA schema v2 (#3)
Browse files Browse the repository at this point in the history 
* add support for sources tables

* handle custom schema, fix insert code for contracts and compiled_contracts

* remove log

* fix PR suggestion
  • Loading branch information
@marcocastignoli
marcocastignoli authored Oct 22, 2024
1 parent 96d93dd commit 3440c2d
Show file tree
Hide file tree
Showing 2 changed files with 202 additions and 65 deletions.
2 changes: 2 additions & 0 deletions .env.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@ SOURCIFY_DB=
SOURCIFY_USER=
SOURCIFY_PASSWORD=
SOURCIFY_PORT=
SOURCIFY_SCHEMA=

VERA_HOST=
VERA_DB=
VERA_USER=
VERA_PASSWORD=
VERA_PORT=
VERA_SCHEMA=

SOURCIFY_SERVER_HOST=
265 changes: 200 additions & 65 deletions pushToVera.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ if (fs.existsSync(COUNTER_FILE)) {

const N = 200; // Number of contracts to process at a time

const SOURCIFY_SCHEMA = process.env.SOURCIFY_SCHEMA || "public";
const VERA_SCHEMA = process.env.VERA_SCHEMA || "public";

const SOURCE_DB_CONFIG = {
host: process.env.SOURCIFY_HOST,
database: process.env.SOURCIFY_DB,
Expand Down Expand Up @@ -71,29 +74,30 @@ async function upsertAndGetId(

const { rows: countLeft } = await sourceClient.query(
`
SELECT count(vc.*) FROM sourcify_matches sm
JOIN verified_contracts vc ON vc.id = sm.verified_contract_id
JOIN contract_deployments cd on vc.deployment_id = cd.id
JOIN contracts c on cd.contract_id = c.id
JOIN code on code.code_hash = c.creation_code_hash
WHERE 1=1
and sm.creation_match is not null
and sm.runtime_match is not null
and cd.transaction_hash is not null
and code.code is not null
and vc.id >= $1;
`,
SELECT count(vc.*)
FROM ${SOURCIFY_SCHEMA}.sourcify_matches sm
JOIN ${SOURCIFY_SCHEMA}.verified_contracts vc ON vc.id = sm.verified_contract_id
JOIN ${SOURCIFY_SCHEMA}.contract_deployments cd on vc.deployment_id = cd.id
JOIN ${SOURCIFY_SCHEMA}.contracts c on cd.contract_id = c.id
JOIN ${SOURCIFY_SCHEMA}.code on code.code_hash = c.creation_code_hash
WHERE 1=1
and sm.creation_match is not null
and sm.runtime_match is not null
and cd.transaction_hash is not null
and code.code is not null
and vc.id >= $1;
`,
[CURRENT_VERIFIED_CONTRACT]
);
console.log("Number of contracts left: ", countLeft[0].count);

const { rows: verifiedContracts, rowCount } = await sourceClient.query(
`
SELECT vc.* FROM sourcify_matches sm
JOIN verified_contracts vc ON vc.id = sm.verified_contract_id
JOIN contract_deployments cd on vc.deployment_id = cd.id
JOIN contracts c on cd.contract_id = c.id
JOIN code on code.code_hash = c.creation_code_hash
SELECT vc.* FROM ${SOURCIFY_SCHEMA}.sourcify_matches sm
JOIN ${SOURCIFY_SCHEMA}.verified_contracts vc ON vc.id = sm.verified_contract_id
JOIN ${SOURCIFY_SCHEMA}.contract_deployments cd on vc.deployment_id = cd.id
JOIN ${SOURCIFY_SCHEMA}.contracts c on cd.contract_id = c.id
JOIN ${SOURCIFY_SCHEMA}.code on code.code_hash = c.creation_code_hash
WHERE 1=1
and sm.creation_match is not null
and sm.runtime_match is not null
Expand All @@ -102,7 +106,7 @@ async function upsertAndGetId(
and vc.id >= $1
ORDER BY vc.id ASC
LIMIT $2;
`,
`,
[CURRENT_VERIFIED_CONTRACT, N]
);

Expand Down Expand Up @@ -130,63 +134,122 @@ async function upsertAndGetId(
const {
rows: [deployment],
} = await sourceClient.query(
"SELECT * FROM contract_deployments WHERE id = $1",
`SELECT * FROM ${SOURCIFY_SCHEMA}.contract_deployments WHERE id = $1`,
[deployment_id]
);
const {
rows: [deploymentContract],
} = await sourceClient.query(
`SELECT * FROM ${SOURCIFY_SCHEMA}.contracts WHERE id = $1`,
[deployment.contract_id]
);
const {
rows: [compilation],
} = await sourceClient.query(
"SELECT * FROM compiled_contracts WHERE id = $1",
`SELECT * FROM ${SOURCIFY_SCHEMA}.compiled_contracts WHERE id = $1`,
[compilation_id]
);
if (compilation.creation_code_hash === null) {
continue;
}

// Get creation and runtime code for the compilation
const {
rows: [creationCode],
rows: [compilationCreationCode],
} = await sourceClient.query(
"SELECT * FROM code WHERE code_hash = $1",
`SELECT * FROM ${SOURCIFY_SCHEMA}.code WHERE code_hash = $1`,
[compilation.creation_code_hash]
);
const {
rows: [runtimeCode],
rows: [compilationRuntimeCode],
} = await sourceClient.query(
"SELECT * FROM code WHERE code_hash = $1",
`SELECT * FROM ${SOURCIFY_SCHEMA}.code WHERE code_hash = $1`,
[compilation.runtime_code_hash]
);

// Get creation and runtime code for the deployment
const {
rows: [deployedCreationCode],
} = await sourceClient.query(
`SELECT * FROM ${SOURCIFY_SCHEMA}.code WHERE code_hash = $1`,
[deploymentContract.creation_code_hash]
);
const {
rows: [deployedRuntimeCode],
} = await sourceClient.query(
`SELECT * FROM ${SOURCIFY_SCHEMA}.code WHERE code_hash = $1`,
[deploymentContract.runtime_code_hash]
);

// Insert dependencies into target DB and handle conflicts

// Insert creation and runtime code for the compilation into target DB and handle conflicts
await targetClient.query(
`
INSERT INTO ${VERA_SCHEMA}.code (code_hash, code, code_hash_keccak)
VALUES ($1, $2, $3)
ON CONFLICT (code_hash) DO NOTHING
`,
[
compilationCreationCode.code_hash,
compilationCreationCode.code,
compilationCreationCode.code_hash_keccak,
]
);

await targetClient.query(
`
INSERT INTO ${VERA_SCHEMA}.code (code_hash, code, code_hash_keccak)
VALUES ($1, $2, $3)
ON CONFLICT (code_hash) DO NOTHING
`,
[
compilationRuntimeCode.code_hash,
compilationRuntimeCode.code,
compilationRuntimeCode.code_hash_keccak,
]
);

// Insert creation and runtime code for the deployment into target DB and handle conflicts
await targetClient.query(
`
INSERT INTO code (code_hash, code)
VALUES ($1, $2)
ON CONFLICT (code_hash) DO NOTHING
`,
[creationCode.code_hash, creationCode.code]
INSERT INTO ${VERA_SCHEMA}.code (code_hash, code, code_hash_keccak)
VALUES ($1, $2, $3)
ON CONFLICT (code_hash) DO NOTHING
`,
[
deployedCreationCode.code_hash,
deployedCreationCode.code,
deployedCreationCode.code_hash_keccak,
]
);

await targetClient.query(
`
INSERT INTO code (code_hash, code)
VALUES ($1, $2)
ON CONFLICT (code_hash) DO NOTHING
`,
[runtimeCode.code_hash, runtimeCode.code]
INSERT INTO ${VERA_SCHEMA}.code (code_hash, code, code_hash_keccak)
VALUES ($1, $2, $3)
ON CONFLICT (code_hash) DO NOTHING
`,
[
deployedRuntimeCode.code_hash,
deployedRuntimeCode.code,
deployedRuntimeCode.code_hash_keccak,
]
);

const newContractIdValues = [
compilation.creation_code_hash,
compilation.runtime_code_hash,
deploymentContract.creation_code_hash,
deploymentContract.runtime_code_hash,
];
const newContractId = await upsertAndGetId(
`
INSERT INTO contracts (creation_code_hash, runtime_code_hash)
VALUES ($1, $2)
ON CONFLICT (creation_code_hash, runtime_code_hash) DO NOTHING
`,
INSERT INTO ${VERA_SCHEMA}.contracts (creation_code_hash, runtime_code_hash)
VALUES ($1, $2)
ON CONFLICT (creation_code_hash, runtime_code_hash) DO NOTHING
`,
`
SELECT id FROM contracts WHERE creation_code_hash = $1 AND runtime_code_hash = $2
`,
SELECT id FROM ${VERA_SCHEMA}.contracts WHERE creation_code_hash = $1 AND runtime_code_hash = $2
`,
newContractIdValues,
newContractIdValues,
targetClient
Expand All @@ -203,13 +266,13 @@ async function upsertAndGetId(
];
const newDeploymentId = await upsertAndGetId(
`
INSERT INTO contract_deployments (chain_id, address, transaction_hash, block_number, transaction_index, deployer, contract_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (chain_id, address, transaction_hash) DO NOTHING
`,
INSERT INTO ${VERA_SCHEMA}.contract_deployments (chain_id, address, transaction_hash, block_number, transaction_index, deployer, contract_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (chain_id, address, transaction_hash) DO NOTHING
`,
`
SELECT id FROM contract_deployments WHERE chain_id = $1 AND address = $2 AND transaction_hash = $3
`,
SELECT id FROM ${VERA_SCHEMA}.contract_deployments WHERE chain_id = $1 AND address = $2 AND transaction_hash = $3
`,
newDeploymentIdValues,
newDeploymentIdValues.slice(0, 3),
targetClient
Expand All @@ -223,37 +286,108 @@ async function upsertAndGetId(
compilation.version,
compilation.name,
compilation.fully_qualified_name,
compilation.sources,
compilation.compiler_settings,
compilation.compilation_artifacts,
compilation.creation_code_artifacts,
compilation.runtime_code_artifacts,
];
const newCompilationId = await upsertAndGetId(
`
INSERT INTO compiled_contracts (
compiler, language, creation_code_hash, runtime_code_hash, version, name, fully_qualified_name, sources, compiler_settings,
compilation_artifacts, creation_code_artifacts, runtime_code_artifacts)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
ON CONFLICT (compiler, language, creation_code_hash, runtime_code_hash) DO NOTHING
`,
INSERT INTO ${VERA_SCHEMA}.compiled_contracts (
compiler, language, creation_code_hash, runtime_code_hash, version, name, fully_qualified_name,
compiler_settings, compilation_artifacts, creation_code_artifacts, runtime_code_artifacts)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT (compiler, language, creation_code_hash, runtime_code_hash) DO NOTHING
`,
`
SELECT id FROM compiled_contracts WHERE compiler = $1 AND language = $2 AND creation_code_hash = $3 AND runtime_code_hash = $4
`,
SELECT id FROM ${VERA_SCHEMA}.compiled_contracts
WHERE compiler = $1 AND language = $2 AND creation_code_hash = $3 AND runtime_code_hash = $4
`,
newCompilationIdValues,
newCompilationIdValues.slice(0, 4),
targetClient
);

// Get sources for the compilation
const { rows: compilationSources } = await sourceClient.query(
`SELECT * FROM ${SOURCIFY_SCHEMA}.compiled_contracts_sources WHERE compilation_id = $1`,
[compilation_id]
);

// Extract all source_hashes
const sourceHashes = compilationSources.map(
(source) => source.source_hash
);

// Fetch all relevant sources in a single query
const { rows: sourcesContent } = await sourceClient.query(
`SELECT * FROM ${SOURCIFY_SCHEMA}.sources WHERE source_hash = ANY($1)`,
[sourceHashes]
);

// Create a map for quick lookup
const sourceContentMap = new Map(
sourcesContent.map((source) => [
source.source_hash.toString("hex"),
source,
])
);

// Insert sources and compiled_contracts_sources
for (const source of compilationSources) {
const sourceContent = sourceContentMap.get(
source.source_hash.toString("hex")
);

if (!sourceContent) {
console.warn(
`Source content not found for hash: ${source.source_hash.toString(
"hex"
)}`
);
continue;
}

// Insert into sources table
const sourceValues = [
Buffer.from(sourceContent.source_hash),
Buffer.from(sourceContent.source_hash_keccak),
sourceContent.content,
];
await targetClient.query(
`
INSERT INTO ${VERA_SCHEMA}.sources (source_hash, source_hash_keccak, content)
VALUES ($1, $2, $3)
ON CONFLICT (source_hash) DO NOTHING
`,
sourceValues
);

// Insert into compiled_contracts_sources table
const compiledSourcesValues = [
newCompilationId,
Buffer.from(source.source_hash),
source.path,
];
await targetClient.query(
`
INSERT INTO ${VERA_SCHEMA}.compiled_contracts_sources (compilation_id, source_hash, path)
VALUES ($1, $2, $3)
ON CONFLICT (compilation_id, path) DO NOTHING
`,
compiledSourcesValues
);
}

await targetClient.query(
`
INSERT INTO verified_contracts (
created_at, updated_at, created_by, updated_by, deployment_id, compilation_id,
creation_match, creation_values, creation_transformations,
runtime_match, runtime_values, runtime_transformations)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
ON CONFLICT (compilation_id, deployment_id) DO NOTHING
`,
INSERT INTO ${VERA_SCHEMA}.verified_contracts (
created_at, updated_at, created_by, updated_by, deployment_id, compilation_id,
creation_match, creation_values, creation_transformations, creation_metadata_match,
runtime_match, runtime_values, runtime_transformations, runtime_metadata_match)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
ON CONFLICT (compilation_id, deployment_id) DO NOTHING
`,
[
verifiedContract.created_at,
verifiedContract.updated_at,
Expand All @@ -264,9 +398,11 @@ async function upsertAndGetId(
verifiedContract.creation_match,
verifiedContract.creation_values,
JSON.stringify(verifiedContract.creation_transformations),
verifiedContract.creation_metadata_match,
verifiedContract.runtime_match,
verifiedContract.runtime_values,
JSON.stringify(verifiedContract.runtime_transformations),
verifiedContract.runtime_metadata_match,
]
);
}
Expand All @@ -278,7 +414,6 @@ async function upsertAndGetId(
N / (iterationTimeTaken / 1000)
} contracts per second`
);
console.log();
}
console.log("Contracts transferred successfully.");
} catch (error) {
Expand Down

0 comments on commit 3440c2d

Please sign in to comment.