From d78668222c750b5ccdd01aeab38985289a0b93af Mon Sep 17 00:00:00 2001 From: portuu3 <> Date: Thu, 23 Jul 2026 09:58:09 +0200 Subject: [PATCH 1/3] update cvat payout calculator for the new manifest --- .../cvat-payouts-calculator.spec.ts | 96 +++++++++++++------ .../cvat-payouts-calculator.ts | 49 ++++------ 2 files changed, 82 insertions(+), 63 deletions(-) diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.spec.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.spec.ts index 00cf233469..8deae1bde8 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.spec.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.spec.ts @@ -4,7 +4,6 @@ import { faker } from '@faker-js/faker'; import { createMock } from '@golevelup/ts-jest'; import { EscrowClient } from '@human-protocol/sdk'; import { Test } from '@nestjs/testing'; -import { ethers } from 'ethers'; import _ from 'lodash'; import { CvatAnnotationMeta } from '@/common/types'; @@ -48,14 +47,24 @@ describe('CvatPayoutsCalculator', () => { const mockedGetIntermediateResultsUrl = jest .fn() .mockImplementation(async () => faker.internet.url()); - const mockedGetTokenAddress = jest.fn().mockImplementation(async () => { - return faker.finance.ethereumAddress(); - }); + const mockedGetReservedFunds = jest + .fn() + .mockImplementation(async () => + BigInt(faker.number.int({ min: 1000, max: 100000 })), + ); beforeAll(() => { mockedEscrowClient.build.mockResolvedValue({ getIntermediateResultsUrl: mockedGetIntermediateResultsUrl, - getTokenAddress: mockedGetTokenAddress, + getReservedFunds: mockedGetReservedFunds, + } as unknown as EscrowClient); + }); + + afterEach(() => { + jest.resetAllMocks(); + mockedEscrowClient.build.mockResolvedValue({ + getIntermediateResultsUrl: mockedGetIntermediateResultsUrl, + getReservedFunds: mockedGetReservedFunds, } as unknown as EscrowClient); }); @@ -84,23 +93,22 @@ describe('CvatPayoutsCalculator', () => { ); }); - it('should properly calculate workers bounties trimming the decimals', async () => { + it('should properly calculate workers bounties', async () => { const annotators = [ faker.finance.ethereumAddress(), faker.finance.ethereumAddress(), ]; const jobsPerAnnotator = faker.number.int({ min: 1, max: 3 }); - const tokenDecimals = BigInt(6); + const jobCount = jobsPerAnnotator * annotators.length; + const payoutPerJob = BigInt(faker.number.int({ min: 1000, max: 100000 })); + const reservedFunds = payoutPerJob * BigInt(jobCount); const annotationsMeta: CvatAnnotationMeta = { - jobs: Array.from( - { length: jobsPerAnnotator * annotators.length }, - (_v, index: number) => ({ - job_id: index, - final_result_id: faker.number.int(), - }), - ), + jobs: Array.from({ length: jobCount }, (_v, index: number) => ({ + job_id: index, + final_result_id: faker.number.int(), + })), results: [], }; for (const job of annotationsMeta.jobs) { @@ -130,26 +138,19 @@ describe('CvatPayoutsCalculator', () => { mockedStorageService.downloadJsonLikeData.mockResolvedValueOnce( annotationsMeta, ); - mockedWeb3Service.getTokenDecimals.mockResolvedValueOnce(tokenDecimals); - - const mockedJobBounty = '0.123456789'; // more decimals than token has - const manifest = { - ...generateCvatManifest(), - job_bounty: mockedJobBounty, - }; + mockedGetReservedFunds.mockResolvedValueOnce(reservedFunds); + const manifest = generateCvatManifest(); const payouts = await calculator.calculate({ chainId, escrowAddress, - manifest, + manifest: manifest, finalResultsUrl: faker.internet.url(), }); - const trimmedBounty = '0.123456'; - + const expectedJobBounty = payoutPerJob; const expectedAmountPerAnnotator = - BigInt(jobsPerAnnotator) * - ethers.parseUnits(trimmedBounty, tokenDecimals); + BigInt(jobsPerAnnotator) * expectedJobBounty; const expectedPayouts = annotators.map((address) => ({ address, @@ -168,7 +169,9 @@ describe('CvatPayoutsCalculator', () => { ]; const jobsPerAnnotator = faker.number.int({ min: 1, max: 3 }); - const tokenDecimals = BigInt(faker.number.int({ min: 6, max: 18 })); + const reservedFunds = BigInt( + faker.number.int({ min: 1000, max: 100000 }).toString(), + ); const annotationsMeta: CvatAnnotationMeta = { jobs: Array.from( @@ -207,7 +210,7 @@ describe('CvatPayoutsCalculator', () => { mockedStorageService.downloadJsonLikeData.mockResolvedValueOnce( annotationsMeta, ); - mockedWeb3Service.getTokenDecimals.mockResolvedValueOnce(tokenDecimals); + mockedGetReservedFunds.mockResolvedValueOnce(reservedFunds); const manifest = generateCvatManifest(); @@ -218,9 +221,14 @@ describe('CvatPayoutsCalculator', () => { finalResultsUrl: faker.internet.url(), }); + const matchedJobCount = annotationsMeta.jobs.filter((job) => + annotationsMeta.results.some( + (result) => result.id === job.final_result_id, + ), + ).length; + const expectedJobBounty = reservedFunds / BigInt(matchedJobCount); const expectedAmountPerAnnotator = - BigInt(jobsPerAnnotator) * - ethers.parseUnits(manifest.job_bounty, tokenDecimals); + BigInt(jobsPerAnnotator) * expectedJobBounty; const expectedPayouts = annotators.map((address) => ({ address, @@ -231,5 +239,33 @@ describe('CvatPayoutsCalculator', () => { _.sortBy(expectedPayouts, 'address'), ); }); + + it('throws when there are no jobs with matching final results', async () => { + mockedStorageService.downloadJsonLikeData.mockResolvedValueOnce({ + jobs: [ + { + job_id: faker.number.int(), + final_result_id: faker.number.int(), + }, + ], + results: [ + { + id: faker.number.int(), + job_id: faker.number.int(), + annotator_wallet_address: faker.finance.ethereumAddress(), + annotation_quality: faker.number.float(), + }, + ], + } as CvatAnnotationMeta); + + await expect( + calculator.calculate({ + chainId, + escrowAddress, + manifest: generateCvatManifest(), + finalResultsUrl: faker.internet.url(), + }), + ).rejects.toThrow('Invalid annotation meta'); + }); }); }); diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.ts index bde421dbca..0546299622 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.ts @@ -1,6 +1,5 @@ import { EscrowClient } from '@human-protocol/sdk'; import { Injectable } from '@nestjs/common'; -import { ethers } from 'ethers'; import type { OverrideProperties } from 'type-fest'; import { CVAT_VALIDATION_META_FILENAME } from '@/common/constants'; @@ -27,7 +26,6 @@ export class CvatPayoutsCalculator implements EscrowPayoutsCalculator { ) {} async calculate({ - manifest, chainId, escrowAddress, }: CalculateCvatPayoutsInput): Promise { @@ -41,26 +39,28 @@ export class CvatPayoutsCalculator implements EscrowPayoutsCalculator { await this.storageService.downloadJsonLikeData( `${intermediateResultsUrl}/${CVAT_VALIDATION_META_FILENAME}`, ); - if (!annotations.jobs.length || !annotations.results.length) { throw new Error('Invalid annotation meta'); } - const tokenAddress = await escrowClient.getTokenAddress(escrowAddress); - const tokenDecimals = await this.web3Service.getTokenDecimals( - chainId, - tokenAddress, - ); - const jobBountyValue = this.parseJobBounty( - manifest.job_bounty, - Number(tokenDecimals), - ); - const workersBounties = new Map(); + const reservedFunds = await escrowClient.getReservedFunds(escrowAddress); - for (const job of annotations.jobs) { - const jobFinalResult = annotations.results.find( - (result) => result.id === job.final_result_id, + const matchedJobResults = annotations.jobs + .map((job) => + annotations.results.find((result) => result.id === job.final_result_id), + ) + .filter((result): result is CvatAnnotationMeta['results'][number] => + Boolean(result), ); + + if (!matchedJobResults.length) { + throw new Error('Invalid annotation meta'); + } + + const jobBountyValue = reservedFunds / BigInt(matchedJobResults.length); + const workersBounties = new Map(); + + for (const jobFinalResult of matchedJobResults) { // TODO: enable annotation quality validation when ready if ( jobFinalResult @@ -84,21 +84,4 @@ export class CvatPayoutsCalculator implements EscrowPayoutsCalculator { }), ); } - - private parseJobBounty(jobBounty: string, tokenDecimals: number): bigint { - const parts = jobBounty.split('.'); - if (parts.length > 1) { - const decimalsInBounty = parts[1].length; - if (decimalsInBounty > tokenDecimals) { - if (tokenDecimals === 0) { - return ethers.parseUnits(parts[0], tokenDecimals); - } - return ethers.parseUnits( - `${parts[0]}.${parts[1].slice(0, tokenDecimals)}`, - tokenDecimals, - ); - } - } - return ethers.parseUnits(jobBounty, tokenDecimals); - } } From 5f025bca3dd5b66355d92c5c95220dd8b07a3bdc Mon Sep 17 00:00:00 2001 From: portuu3 <> Date: Thu, 23 Jul 2026 16:57:08 +0200 Subject: [PATCH 2/3] fix pr comments --- .../cvat-payouts-calculator.spec.ts | 20 ++++++------------- .../cvat-payouts-calculator.ts | 14 +++---------- 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.spec.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.spec.ts index 8deae1bde8..13c1da8dac 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.spec.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.spec.ts @@ -60,14 +60,6 @@ describe('CvatPayoutsCalculator', () => { } as unknown as EscrowClient); }); - afterEach(() => { - jest.resetAllMocks(); - mockedEscrowClient.build.mockResolvedValue({ - getIntermediateResultsUrl: mockedGetIntermediateResultsUrl, - getReservedFunds: mockedGetReservedFunds, - } as unknown as EscrowClient); - }); - it('throws when invalid annotation meta downloaded from valid url', async () => { const intermediateResultsUrl = faker.internet.url(); mockedGetIntermediateResultsUrl.mockResolvedValueOnce( @@ -102,7 +94,6 @@ describe('CvatPayoutsCalculator', () => { const jobsPerAnnotator = faker.number.int({ min: 1, max: 3 }); const jobCount = jobsPerAnnotator * annotators.length; const payoutPerJob = BigInt(faker.number.int({ min: 1000, max: 100000 })); - const reservedFunds = payoutPerJob * BigInt(jobCount); const annotationsMeta: CvatAnnotationMeta = { jobs: Array.from({ length: jobCount }, (_v, index: number) => ({ @@ -139,12 +130,13 @@ describe('CvatPayoutsCalculator', () => { annotationsMeta, ); - mockedGetReservedFunds.mockResolvedValueOnce(reservedFunds); - const manifest = generateCvatManifest(); + mockedGetReservedFunds.mockResolvedValueOnce( + payoutPerJob * BigInt(jobCount), + ); const payouts = await calculator.calculate({ chainId, escrowAddress, - manifest: manifest, + manifest: generateCvatManifest(), finalResultsUrl: faker.internet.url(), }); @@ -245,12 +237,12 @@ describe('CvatPayoutsCalculator', () => { jobs: [ { job_id: faker.number.int(), - final_result_id: faker.number.int(), + final_result_id: 1, }, ], results: [ { - id: faker.number.int(), + id: 2, job_id: faker.number.int(), annotator_wallet_address: faker.finance.ethereumAddress(), annotation_quality: faker.number.float(), diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.ts index 0546299622..ad4e1660a8 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.ts @@ -62,19 +62,11 @@ export class CvatPayoutsCalculator implements EscrowPayoutsCalculator { for (const jobFinalResult of matchedJobResults) { // TODO: enable annotation quality validation when ready - if ( - jobFinalResult - // && jobFinalResult.annotation_quality >= manifest.validation.min_quality - ) { - const workerAddress = jobFinalResult.annotator_wallet_address; + const workerAddress = jobFinalResult.annotator_wallet_address; - const currentWorkerBounty = workersBounties.get(workerAddress) || 0n; + const currentWorkerBounty = workersBounties.get(workerAddress) || 0n; - workersBounties.set( - workerAddress, - currentWorkerBounty + jobBountyValue, - ); - } + workersBounties.set(workerAddress, currentWorkerBounty + jobBountyValue); } return Array.from(workersBounties.entries()).map( From 8e7671459aeb84ad1c5b3a3a78e6bc520a07e59b Mon Sep 17 00:00:00 2001 From: portuu3 <> Date: Mon, 27 Jul 2026 11:46:46 +0200 Subject: [PATCH 3/3] test mock improvement --- .../payouts-calculation/default-payouts-calculator.spec.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/default-payouts-calculator.spec.ts b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/default-payouts-calculator.spec.ts index 7f7e79cb4e..6f5c574f4a 100644 --- a/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/default-payouts-calculator.spec.ts +++ b/packages/apps/reputation-oracle/server/src/modules/escrow-completion/payouts-calculation/default-payouts-calculator.spec.ts @@ -49,10 +49,7 @@ describe('DefaultPayoutsCalculator', () => { }); describe('calculate', () => { - const reservedFunds = BigInt(faker.number.int({ min: 1000 }).toString()); - const mockedGetReservedFunds = jest - .fn() - .mockImplementation(async () => reservedFunds); + const mockedGetReservedFunds = jest.fn().mockResolvedValue(0n); beforeEach(() => { mockedEscrowClient.build.mockResolvedValue({ @@ -74,7 +71,9 @@ describe('DefaultPayoutsCalculator', () => { requestType: MarketingJobType.SOCIAL_MEDIA_PROMOTION, submissionsRequired: faker.number.int({ min: 2, max: 5 }), }; + const reservedFunds = BigInt(faker.number.int({ min: 1000 }).toString()); + mockedGetReservedFunds.mockResolvedValueOnce(reservedFunds); mockedStorageService.downloadJsonLikeData.mockResolvedValueOnce(results); const payouts = await calculator.calculate({