feat: Idempotent Provisioning (#1004)

Provisioning is occasionally finnicky, failing on different parts of the process. Having retryable provisioning allows us to heal indexer state through retries rather than manually actioning this. To this, the provisioning step needs to have access to accurate status of provisioning in order to make the correct decisions on which non-idempotent actions to take. This PR introduces a ProvisioningState class which represents the status of provisioning, and can be used to make decisions on what parts of provisioning need to be completed.
near · Aug 14, 2024 · 5633b4a · 5633b4a
1 parent 8dd78c7
commit 5633b4a
Show file tree

Hide file tree

Showing 18 changed files with 753 additions and 135 deletions.
diff --git a/.../__snapshots__/hasura-client.test.ts.snap → .../__snapshots__/hasura-client.test.ts.snap b/.../__snapshots__/hasura-client.test.ts.snap → .../__snapshots__/hasura-client.test.ts.snap
diff --git a/...r/src/hasura-client/hasura-client.test.ts → ...ioner/hasura-client/hasura-client.test.ts b/...r/src/hasura-client/hasura-client.test.ts → ...ioner/hasura-client/hasura-client.test.ts
diff --git a/runner/src/hasura-client/hasura-client.ts → ...rovisioner/hasura-client/hasura-client.ts b/runner/src/hasura-client/hasura-client.ts → ...rovisioner/hasura-client/hasura-client.ts
@@ -10,7 +10,7 @@ interface SqlOptions {
   source?: string
 }
 
-export type HasuraPermission = 'select' | 'insert' | 'update' | 'delete';
+export const HASURA_PERMISSION_TYPES = ['select', 'insert', 'update', 'delete'];
 
 interface TableDefinition {
   name: string

diff --git a/runner/src/hasura-client/index.ts → ...er/src/provisioner/hasura-client/index.ts b/runner/src/hasura-client/index.ts → ...er/src/provisioner/hasura-client/index.ts
@@ -1,2 +1,3 @@
 export { default } from './hasura-client';
-export type { HasuraMetadata, HasuraSource, HasuraConfiguration, HasuraDatabaseConnectionParameters, HasuraTableMetadata, HasuraRolePermission, HasuraPermission } from './hasura-client';
+export type { HasuraMetadata, HasuraSource, HasuraConfiguration, HasuraDatabaseConnectionParameters, HasuraTableMetadata, HasuraRolePermission } from './hasura-client';
+export { HASURA_PERMISSION_TYPES } from './hasura-client';
diff --git a/runner/src/provisioner/index.ts b/runner/src/provisioner/index.ts
@@ -1 +1,2 @@
 export { default } from './provisioner';
+export { METADATA_TABLE_NAME, LOGS_TABLE_NAME } from './provisioner';
diff --git a/runner/src/provisioner/provisioner.test.ts b/runner/src/provisioner/provisioner.test.ts
@@ -1,8 +1,9 @@
 import pgFormat from 'pg-format';
 
-import Provisioner from './provisioner';
+import Provisioner, { LOGS_TABLE_NAME, METADATA_TABLE_NAME } from './provisioner';
 import IndexerConfig from '../indexer-config/indexer-config';
 import { LogLevel } from '../indexer-meta/log-entry';
+import { type HasuraTableMetadata, type HasuraMetadata, type HasuraSource } from './hasura-client';
 
 describe('Provisioner', () => {
   let adminPgClient: any;
@@ -13,15 +14,24 @@ describe('Provisioner', () => {
   let indexerConfig: IndexerConfig;
 
   const tableNames = ['blocks'];
+  const systemTables = [METADATA_TABLE_NAME, LOGS_TABLE_NAME];
+  const tableNamesWithSystemTables = ['blocks', ...systemTables];
   const accountId = 'morgs.near';
   const functionName = 'test-function';
   const databaseSchema = 'CREATE TABLE blocks (height numeric)';
   indexerConfig = new IndexerConfig('', accountId, functionName, 0, '', databaseSchema, LogLevel.INFO);
+  const emptyHasuraMetadata = generateDefaultHasuraMetadata();
+  const hasuraMetadataWithEmptySource = generateDefaultHasuraMetadata();
+  hasuraMetadataWithEmptySource.sources.push(generateSourceWithTables([], [], indexerConfig.userName(), indexerConfig.databaseName()));
+  const hasuraMetadataWithSystemProvisions = generateDefaultHasuraMetadata();
+  hasuraMetadataWithSystemProvisions.sources.push(generateSourceWithTables([indexerConfig.schemaName()], systemTables, indexerConfig.userName(), indexerConfig.databaseName()));
+  const hasuraMetadataWithProvisions = generateDefaultHasuraMetadata();
+  hasuraMetadataWithProvisions.sources.push(generateSourceWithTables([indexerConfig.schemaName()], tableNamesWithSystemTables, indexerConfig.userName(), indexerConfig.databaseName()));
   const testingRetryConfig = {
     maxRetries: 5,
     baseDelay: 10
   };
-  const setProvisioningStatusQuery = `INSERT INTO ${indexerConfig.schemaName()}.sys_metadata (attribute, value) VALUES ('STATUS', 'PROVISIONING') ON CONFLICT (attribute) DO UPDATE SET value = EXCLUDED.value RETURNING *`;
+  const setProvisioningStatusQuery = `INSERT INTO ${indexerConfig.schemaName()}.${METADATA_TABLE_NAME} (attribute, value) VALUES ('STATUS', 'PROVISIONING') ON CONFLICT (attribute) DO UPDATE SET value = EXCLUDED.value RETURNING *`;
   const logsDDL = expect.any(String);
   const metadataDDL = expect.any(String);
   const error = new Error('some error');
@@ -39,7 +49,8 @@ describe('Provisioner', () => {
 
   beforeEach(() => {
     hasuraClient = {
-      getTableNames: jest.fn().mockReturnValueOnce(tableNames),
+      exportMetadata: jest.fn().mockResolvedValueOnce(emptyHasuraMetadata).mockResolvedValue(hasuraMetadataWithSystemProvisions),
+      getTableNames: jest.fn().mockResolvedValueOnce([]).mockResolvedValue(tableNamesWithSystemTables),
       trackTables: jest.fn().mockReturnValueOnce(null),
       trackForeignKeyRelationships: jest.fn().mockReturnValueOnce(null),
       addPermissionsToTables: jest.fn().mockReturnValueOnce(null),
@@ -179,28 +190,6 @@ describe('Provisioner', () => {
     });
   });
 
-  describe('isUserApiProvisioned', () => {
-    it('returns false if datasource doesnt exists', async () => {
-      hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(false);
-
-      await expect(provisioner.isProvisioned(indexerConfig)).resolves.toBe(false);
-    });
-
-    it('returns false if datasource and schema dont exists', async () => {
-      hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(false);
-      hasuraClient.doesSchemaExist = jest.fn().mockReturnValueOnce(false);
-
-      await expect(provisioner.isProvisioned(indexerConfig)).resolves.toBe(false);
-    });
-
-    it('returns true if datasource and schema exists', async () => {
-      hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(true);
-      hasuraClient.doesSchemaExist = jest.fn().mockReturnValueOnce(true);
-
-      await expect(provisioner.isProvisioned(indexerConfig)).resolves.toBe(true);
-    });
-  });
-
   describe('provisionUserApi', () => {
     it('provisions an API for the user', async () => {
       await provisioner.provisionUserApi(indexerConfig);
@@ -227,7 +216,20 @@ describe('Provisioner', () => {
       expect(hasuraClient.executeSqlOnSchema).toHaveBeenNthCalledWith(2, indexerConfig.userName(), indexerConfig.schemaName(), logsDDL);
       expect(hasuraClient.executeSqlOnSchema).toHaveBeenNthCalledWith(3, indexerConfig.userName(), indexerConfig.schemaName(), databaseSchema);
       expect(hasuraClient.getTableNames).toBeCalledWith(indexerConfig.schemaName(), indexerConfig.databaseName());
-      expect(hasuraClient.trackTables).toBeCalledWith(indexerConfig.schemaName(), tableNames, indexerConfig.databaseName());
+      expect(hasuraClient.trackTables).toHaveBeenNthCalledWith(1, indexerConfig.schemaName(), [METADATA_TABLE_NAME, LOGS_TABLE_NAME], indexerConfig.databaseName());
+      expect(hasuraClient.trackTables).toHaveBeenNthCalledWith(2, indexerConfig.schemaName(), tableNames, indexerConfig.databaseName());
+      expect(hasuraClient.addPermissionsToTables).toBeCalledWith(
+        indexerConfig.schemaName(),
+        indexerConfig.databaseName(),
+        [METADATA_TABLE_NAME, LOGS_TABLE_NAME],
+        indexerConfig.userName(),
+        [
+          'select',
+          'insert',
+          'update',
+          'delete'
+        ]
+      );
       expect(hasuraClient.addPermissionsToTables).toBeCalledWith(
         indexerConfig.schemaName(),
         indexerConfig.databaseName(),
@@ -243,7 +245,7 @@ describe('Provisioner', () => {
     });
 
     it('skips provisioning the datasource if it already exists', async () => {
-      hasuraClient.doesSourceExist = jest.fn().mockReturnValueOnce(true);
+      hasuraClient.exportMetadata = jest.fn().mockResolvedValueOnce(hasuraMetadataWithEmptySource).mockResolvedValue(hasuraMetadataWithSystemProvisions);
 
       await provisioner.provisionUserApi(indexerConfig);
 
@@ -255,7 +257,20 @@ describe('Provisioner', () => {
       expect(hasuraClient.executeSqlOnSchema).toHaveBeenNthCalledWith(2, indexerConfig.userName(), indexerConfig.schemaName(), logsDDL);
       expect(hasuraClient.executeSqlOnSchema).toHaveBeenNthCalledWith(3, indexerConfig.databaseName(), indexerConfig.schemaName(), databaseSchema);
       expect(hasuraClient.getTableNames).toBeCalledWith(indexerConfig.schemaName(), indexerConfig.databaseName());
-      expect(hasuraClient.trackTables).toBeCalledWith(indexerConfig.schemaName(), tableNames, indexerConfig.databaseName());
+      expect(hasuraClient.trackTables).toHaveBeenNthCalledWith(1, indexerConfig.schemaName(), [METADATA_TABLE_NAME, LOGS_TABLE_NAME], indexerConfig.databaseName());
+      expect(hasuraClient.trackTables).toHaveBeenNthCalledWith(2, indexerConfig.schemaName(), tableNames, indexerConfig.databaseName());
+      expect(hasuraClient.addPermissionsToTables).toBeCalledWith(
+        indexerConfig.schemaName(),
+        indexerConfig.databaseName(),
+        [METADATA_TABLE_NAME, LOGS_TABLE_NAME],
+        indexerConfig.userName(),
+        [
+          'select',
+          'insert',
+          'update',
+          'delete'
+        ]
+      );
       expect(hasuraClient.addPermissionsToTables).toBeCalledWith(
         indexerConfig.schemaName(),
         indexerConfig.databaseName(),
@@ -270,6 +285,22 @@ describe('Provisioner', () => {
       );
     });
 
+    it('skips all provisioning if all provisioning tasks already done', async () => {
+      hasuraClient.exportMetadata = jest.fn().mockResolvedValue(hasuraMetadataWithProvisions);
+      hasuraClient.getTableNames = jest.fn().mockResolvedValue(tableNamesWithSystemTables);
+
+      await provisioner.provisionUserApi(indexerConfig);
+
+      expect(adminPgClient.query).not.toBeCalled();
+      expect(hasuraClient.addDatasource).not.toBeCalled();
+
+      expect(hasuraClient.createSchema).not.toBeCalled();
+      expect(hasuraClient.executeSqlOnSchema).not.toBeCalled();
+      expect(hasuraClient.trackTables).not.toBeCalled();
+      expect(hasuraClient.trackForeignKeyRelationships).toHaveBeenCalledTimes(1);
+      expect(hasuraClient.addPermissionsToTables).not.toBeCalled();
+    });
+
     it('formats user input before executing the query', async () => {
       await provisioner.createUserDb('morgs_near', 'pass; DROP TABLE users;--', 'databaseName UNION SELECT * FROM users --');
 
@@ -312,12 +343,6 @@ describe('Provisioner', () => {
       await expect(provisioner.runLogsSql(accountId, functionName)).rejects.toThrow('Failed to run logs script: some error');
     });
 
-    it('throws an error when it fails to fetch table names', async () => {
-      hasuraClient.getTableNames = jest.fn().mockRejectedValue(error);
-
-      await expect(provisioner.provisionUserApi(indexerConfig)).rejects.toThrow('Failed to provision endpoint: Failed to fetch table names: some error');
-    });
-
     it('throws an error when it fails to track tables', async () => {
       hasuraClient.trackTables = jest.fn().mockRejectedValue(error);
 
@@ -407,3 +432,52 @@ describe('Provisioner', () => {
     });
   });
 });
+
+function generateDefaultHasuraMetadata (): HasuraMetadata {
+  const sources: HasuraSource[] = [];
+  // Insert default source which has different format than the rest
+  sources.push({
+    name: 'default',
+    kind: 'postgres',
+    tables: [],
+    configuration: {
+      connection_info: {
+        database_url: { from_env: 'HASURA_GRAPHQL_DATABASE_URL' },
+      }
+    }
+  });
+
+  return {
+    version: 3,
+    sources
+  };
+}
+
+function generateSourceWithTables (schemaNames: string[], tableNames: string[], role: string, db: string): HasuraSource {
+  const tables: HasuraTableMetadata[] = [];
+  schemaNames.forEach((schemaName) => {
+    tableNames.forEach((tableName) => {
+      tables.push(generateTableConfig(schemaName, tableName, role));
+    });
+  });
+
+  return {
+    name: db,
+    kind: 'postgres',
+    tables,
+    configuration: {} as any,
+  };
+}
+
+function generateTableConfig (schemaName: string, tableName: string, role: string): HasuraTableMetadata {
+  return {
+    table: {
+      name: tableName,
+      schema: schemaName,
+    },
+    insert_permissions: [{ role, permission: {} }],
+    select_permissions: [{ role, permission: {} }],
+    update_permissions: [{ role, permission: {} }],
+    delete_permissions: [{ role, permission: {} }],
+  };
+}