feat: indexing fees / dips (wip)

Author: pcarranzav

packages/indexer-agent/jest.config.js

@@ -11,6 +11,13 @@ module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
   testPathIgnorePatterns: ['/node_modules/', '/dist/', '.yalc'],
+  transformIgnorePatterns: [
+    // Transform @graphprotocol packages but ignore other node_modules
+    '/node_modules/(?!(@graphprotocol)/)'
+  ],
+  transform: {
+    '^.+\\.ts$': ['ts-jest']  // Only transform .ts files
+  },
   globals: {
     __DATABASE__: {
       host:

packages/indexer-agent/src/__tests__/indexer.ts

@@ -146,6 +146,7 @@ const setup = async () => {
   const network = await Network.create(
     logger,
     networkSpecification,
+    models,
     queryFeeModels,
     graphNode,
     metrics,

packages/indexer-agent/src/agent.ts

@@ -223,6 +223,16 @@ export class Agent {
         { logger, milliseconds: requestIntervalSmall },
         async () => {
           const { network, operator } = this.networkAndOperator
+          if (network.specification.indexerOptions.enableDips) {
+            // There should be a DipsManager in the operator
+            if (!operator.dipsManager) {
+              throw new Error('DipsManager is not available')
+            }
+            logger.trace('Ensuring indexing rules for DIPS', {
+              protocolNetwork: network.specification.networkIdentifier,
+            })
+            await operator.dipsManager.ensureAgreementRules()
+          }
           logger.trace('Fetching indexing rules', {
             protocolNetwork: network.specification.networkIdentifier,
           })

packages/indexer-agent/src/commands/start.ts

@@ -307,6 +307,26 @@ export const start = {
         default: 1,
         group: 'Indexer Infrastructure',
       })
+      .option('enable-dips', {
+        description: 'Whether to enable Indexing Fees (DIPs)',
+        type: 'boolean',
+        default: false,
+        group: 'Indexing Fees ("DIPs")',
+      })
+      .option('dipper-endpoint', {
+        description: 'Gateway endpoint for DIPs receipts',
+        type: 'string',
+        array: false,
+        required: false,
+        group: 'Indexing Fees ("DIPs")',
+      })
+      .option('dips-allocation-amount', {
+        description: 'Amount of GRT to allocate for DIPs',
+        type: 'number',
+        default: 1,
+        required: false,
+        group: 'Indexing Fees ("DIPs")',
+      })
       .check(argv => {
         if (
           !argv['network-subgraph-endpoint'] &&
@@ -334,6 +354,9 @@ export const start = {
         ) {
           return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.'
         }
+        if (argv['enable-dips'] && !argv['dipper-endpoint']) {
+          return 'Invalid --dipper-endpoint provided. Must be provided when --enable-dips is true.'
+        }
         return true
       })
   },
@@ -369,6 +392,10 @@ export async function createNetworkSpecification(
     allocateOnNetworkSubgraph: argv.allocateOnNetworkSubgraph,
     register: argv.register,
     finalityTime: argv.chainFinalizeTime,
+    enableDips: argv.enableDips,
+    dipperEndpoint: argv.dipperEndpoint,
+    dipsAllocationAmount: argv.dipsAllocationAmount,
+    dipsEpochsMargin: argv.dipsEpochsMargin,
   }
 
   const transactionMonitoring = {
@@ -585,6 +612,7 @@ export async function run(
   const network = await Network.create(
     logger,
     networkSpecification,
+    managementModels,
     queryFeeModels,
     graphNode,
     metrics,

packages/indexer-cli/jest.config.js

@@ -13,6 +13,13 @@ module.exports = {
   // The glob patterns Jest uses to detect test files
   testMatch: ['**/?(*.)+(spec|test).ts?(x)'],
   testPathIgnorePatterns: ['/node_modules/', '/dist/', '.yalc'],
+  transformIgnorePatterns: [
+    // Transform @graphprotocol packages but ignore other node_modules
+    '/node_modules/(?!(@graphprotocol)/)'
+  ],
+  transform: {
+    '^.+\\.ts$': ['ts-jest']  // Only transform .ts files
+  },
   testTimeout: 60000,
   globals: {
     __DATABASE__: {

packages/indexer-cli/src/__tests__/util.ts

@@ -87,6 +87,7 @@ export const setup = async () => {
   const network = await Network.create(
     logger,
     testNetworkSpecification,
+    models,
     queryFeeModels,
     graphNode,
     metrics,

packages/indexer-common/jest.config.js

@@ -11,7 +11,13 @@ module.exports = {
   forceExit: true,
   testEnvironment: 'node',
   testPathIgnorePatterns: ['/node_modules/', '/dist/', '/.yalc', 'util.ts'],
-  transformIgnorePatterns: ['!node_modules/'],
+  transformIgnorePatterns: [
+    // Transform @graphprotocol packages but ignore other node_modules
+    '/node_modules/(?!(@graphprotocol)/)'
+  ],
+  transform: {
+    '^.+\\.ts$': ['ts-jest']  // Only transform .ts files
+  },
   globals: {
     __DATABASE__: {
       host: process.env.POSTGRES_TEST_HOST || bail('POSTGRES_TEST_HOST is not defined'),

packages/indexer-common/package.json

@@ -22,8 +22,11 @@
     "clean": "rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo"
   },
   "dependencies": {
+    "@bufbuild/protobuf": "2.2.3",
     "@graphprotocol/common-ts": "2.0.11",
     "@graphprotocol/cost-model": "0.1.18",
+    "@graphprotocol/dips-proto": "0.2.1",
+    "@grpc/grpc-js": "^1.12.6",
     "@semiotic-labs/tap-contracts-bindings": "^1.2.1",
     "@thi.ng/heaps": "1.2.38",
     "@types/lodash.clonedeep": "^4.5.7",

packages/indexer-common/src/allocations/__tests__/tap.test.ts

@@ -7,6 +7,7 @@ import {
   TapSubgraphResponse,
   TapCollector,
   Allocation,
+  defineIndexerManagementModels,
 } from '@graphprotocol/indexer-common'
 import {
   Address,
@@ -43,6 +44,7 @@ const setup = async () => {
   // Clearing the registry prevents duplicate metric registration in the default registry.
   metrics.registry.clear()
   sequelize = await connectDatabase(__DATABASE__)
+  const models = defineIndexerManagementModels(sequelize)
   queryFeeModels = defineQueryFeeModels(sequelize)
   sequelize = await sequelize.sync({ force: true })
 
@@ -56,6 +58,7 @@ const setup = async () => {
   const network = await Network.create(
     logger,
     testNetworkSpecification,
+    models,
     queryFeeModels,
     graphNode,
     metrics,

packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts

@@ -1,4 +1,5 @@
 import {
+  defineIndexerManagementModels,
   defineQueryFeeModels,
   GraphNode,
   Network,
@@ -36,6 +37,7 @@ const setup = async () => {
   // Clearing the registry prevents duplicate metric registration in the default registry.
   metrics.registry.clear()
   sequelize = await connectDatabase(__DATABASE__)
+  const models = defineIndexerManagementModels(sequelize)
   queryFeeModels = defineQueryFeeModels(sequelize)
   sequelize = await sequelize.sync({ force: true })
 
@@ -49,6 +51,7 @@ const setup = async () => {
   const network = await Network.create(
     logger,
     testNetworkSpecification,
+    models,
     queryFeeModels,
     graphNode,
     metrics,

packages/indexer-common/src/allocations/escrow-accounts.ts

@@ -13,6 +13,14 @@ export type EscrowAccountResponse = {
   }[]
 }
 
+export type EscrowSenderResponse = {
+  signer: {
+    sender: {
+      id: string
+    }
+  }
+}
+
 export class EscrowAccounts {
   constructor(private sendersBalances: Map<Address, U256>) {}
 
@@ -65,3 +73,26 @@ export const getEscrowAccounts = async (
   }
   return EscrowAccounts.fromResponse(result.data)
 }
+
+export const getEscrowSenderForSigner = async (
+  tapSubgraph: SubgraphClient,
+  signer: Address,
+): Promise<Address> => {
+  const signerLower = signer.toLowerCase()
+  const result = await tapSubgraph.query<EscrowSenderResponse>(
+    gql`
+      query EscrowAccountQuery($signer: ID!) {
+        signer(id: $signer) {
+          sender {
+            id
+          }
+        }
+      }
+    `,
+    { signer: signerLower },
+  )
+  if (!result.data) {
+    throw `There was an error while querying Tap Subgraph. Errors: ${result.error}`
+  }
+  return toAddress(result.data.signer.sender.id)
+}

packages/indexer-common/src/index.ts

@@ -3,6 +3,7 @@ export * from './allocations'
 export * from './async-cache'
 export * from './errors'
 export * from './indexer-management'
+export * from './indexing-fees'
 export * from './graph-node'
 export * from './operator'
 export * from './network'

packages/indexer-common/src/indexer-management/__tests__/allocations.test.ts

@@ -61,6 +61,7 @@ const setup = async () => {
   const network = await Network.create(
     logger,
     testNetworkSpecification,
+    managementModels,
     queryFeeModels,
     graphNode,
     metrics,

packages/indexer-common/src/indexer-management/__tests__/util.ts

@@ -56,6 +56,7 @@ export const createTestManagementClient = async (
   const network = await Network.create(
     logger,
     networkSpecification,
+    managementModels,
     queryFeeModels,
     graphNode,
     metrics,

packages/indexer-common/src/indexer-management/allocations.ts

@@ -15,6 +15,7 @@ import {
   AllocationStatus,
   CloseAllocationResult,
   CreateAllocationResult,
+  DipsManager,
   fetchIndexingRules,
   GraphNode,
   indexerError,
@@ -98,12 +99,23 @@ export type TransactionResult =
   | ActionFailure[]
 
 export class AllocationManager {
+  declare dipsManager: DipsManager | null
   constructor(
     private logger: Logger,
     private models: IndexerManagementModels,
     private graphNode: GraphNode,
     private network: Network,
-  ) {}
+  ) {
+    if (this.network.specification.indexerOptions.dipperEndpoint) {
+      this.dipsManager = new DipsManager(
+        this.logger,
+        this.models,
+        this.graphNode,
+        this.network,
+        this,
+      )
+    }
+  }
 
   async executeBatch(actions: Action[]): Promise<AllocationResult[]> {
     const logger = this.logger.child({ function: 'executeBatch' })
@@ -511,6 +523,14 @@ export class AllocationManager {
       await upsertIndexingRule(logger, this.models, indexingRule)
     }
 
+    if (this.dipsManager) {
+      await this.dipsManager.tryUpdateAgreementAllocation(
+        deployment,
+        null,
+        createAllocationEventLogs.allocationID,
+      )
+    }
+
     return {
       actionID,
       type: 'allocate',
@@ -667,6 +687,15 @@ export class AllocationManager {
 
     await upsertIndexingRule(logger, this.models, neverIndexingRule)
 
+    if (this.dipsManager) {
+      await this.dipsManager.tryCancelAgreement(allocationID)
+      await this.dipsManager.tryUpdateAgreementAllocation(
+        allocation.subgraphDeployment.id.toString(),
+        allocationID,
+        null,
+      )
+    }
+
     return {
       actionID,
       type: 'unallocate',
@@ -966,6 +995,14 @@ export class AllocationManager {
       await upsertIndexingRule(logger, this.models, indexingRule)
     }
 
+    if (this.dipsManager) {
+      await this.dipsManager.tryUpdateAgreementAllocation(
+        subgraphDeploymentID.toString(),
+        allocationID,
+        createAllocationEventLogs.allocationID,
+      )
+    }
+
     return {
       actionID,
       type: 'reallocate',

packages/indexer-common/src/indexer-management/client.ts

@@ -454,9 +454,11 @@ export interface IndexerManagementClientOptions {
   graphNode: GraphNode
   network: Network
   defaults: IndexerManagementDefaults
+  actionManager?: ActionManager | undefined
 }
 
 export class IndexerManagementClient extends Client {
+  declare actionManager: ActionManager | undefined
   private logger?: Logger
   private models: IndexerManagementModels
 
@@ -465,6 +467,7 @@ export class IndexerManagementClient extends Client {
 
     this.logger = options.logger
     this.models = options.models
+    this.actionManager = options.actionManager
   }
 }
 
@@ -501,5 +504,8 @@ export const createIndexerManagementClient = async (
     context,
   })
 
-  return new IndexerManagementClient({ url: 'no-op', exchanges: [exchange] }, options)
+  return new IndexerManagementClient(
+    { url: 'no-op', exchanges: [exchange] },
+    { ...options, actionManager },
+  )
 }

packages/indexer-common/src/indexer-management/models/index.ts

@@ -4,6 +4,7 @@ import { IndexingRuleModels, defineIndexingRuleModels } from './indexing-rule'
 import { CostModelModels, defineCostModelModels } from './cost-model'
 import { POIDisputeModels, definePOIDisputeModels } from './poi-dispute'
 import { ActionModels, defineActionModels } from './action'
+import { defineIndexingFeesModels, IndexingFeesModels } from './indexing-agreement'
 
 export * from './cost-model'
 export * from './indexing-rule'
@@ -13,7 +14,8 @@ export * from './action'
 export type IndexerManagementModels = IndexingRuleModels &
   CostModelModels &
   POIDisputeModels &
-  ActionModels
+  ActionModels &
+  IndexingFeesModels
 
 export const defineIndexerManagementModels = (
   sequelize: Sequelize,
@@ -24,4 +26,5 @@ export const defineIndexerManagementModels = (
     defineIndexingRuleModels(sequelize),
     definePOIDisputeModels(sequelize),
     defineActionModels(sequelize),
+    defineIndexingFeesModels(sequelize),
   )