Skip to content

Update cvat payout calculator for the new manifest #3981

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
portuu3 merged 3 commits into from
Jul 27, 2026
Merged

Update cvat payout calculator for the new manifest #3981

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d786682
update cvat payout calculator for the new manifest
Jul 23, 2026
5f025bc
fix pr comments
Jul 23, 2026
8e76714
test mock improvement
Jul 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 58 additions & 30 deletions .../server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import { faker } from '@faker-js/faker';
import { createMock } from '@golevelup/ts-jest';
import { EscrowClient } from '@human-protocol/sdk';
import { Test } from '@nestjs/testing';
import { ethers } from 'ethers';
import _ from 'lodash';

import { CvatAnnotationMeta } from '@/common/types';
Expand Down Expand Up @@ -48,14 +47,16 @@ describe('CvatPayoutsCalculator', () => {
const mockedGetIntermediateResultsUrl = jest
.fn()
.mockImplementation(async () => faker.internet.url());
const mockedGetTokenAddress = jest.fn().mockImplementation(async () => {
return faker.finance.ethereumAddress();
});
const mockedGetReservedFunds = jest
.fn()
.mockImplementation(async () =>
BigInt(faker.number.int({ min: 1000, max: 100000 })),
Comment thread
dnechay marked this conversation as resolved.
);

beforeAll(() => {
mockedEscrowClient.build.mockResolvedValue({
getIntermediateResultsUrl: mockedGetIntermediateResultsUrl,
getTokenAddress: mockedGetTokenAddress,
getReservedFunds: mockedGetReservedFunds,
} as unknown as EscrowClient);
});

Expand Down Expand Up @@ -84,23 +85,21 @@ describe('CvatPayoutsCalculator', () => {
);
});

it('should properly calculate workers bounties trimming the decimals', async () => {
it('should properly calculate workers bounties', async () => {
const annotators = [
faker.finance.ethereumAddress(),
faker.finance.ethereumAddress(),
];

const jobsPerAnnotator = faker.number.int({ min: 1, max: 3 });
const tokenDecimals = BigInt(6);
const jobCount = jobsPerAnnotator * annotators.length;
const payoutPerJob = BigInt(faker.number.int({ min: 1000, max: 100000 }));

const annotationsMeta: CvatAnnotationMeta = {
jobs: Array.from(
{ length: jobsPerAnnotator * annotators.length },
(_v, index: number) => ({
job_id: index,
final_result_id: faker.number.int(),
}),
),
jobs: Array.from({ length: jobCount }, (_v, index: number) => ({
job_id: index,
final_result_id: faker.number.int(),
})),
results: [],
};
for (const job of annotationsMeta.jobs) {
Expand Down Expand Up @@ -130,26 +129,20 @@ describe('CvatPayoutsCalculator', () => {
mockedStorageService.downloadJsonLikeData.mockResolvedValueOnce(
annotationsMeta,
);
mockedWeb3Service.getTokenDecimals.mockResolvedValueOnce(tokenDecimals);

const mockedJobBounty = '0.123456789'; // more decimals than token has
const manifest = {
...generateCvatManifest(),
job_bounty: mockedJobBounty,
};

mockedGetReservedFunds.mockResolvedValueOnce(
payoutPerJob * BigInt(jobCount),
);
const payouts = await calculator.calculate({
chainId,
escrowAddress,
manifest,
manifest: generateCvatManifest(),
finalResultsUrl: faker.internet.url(),
});

const trimmedBounty = '0.123456';

const expectedJobBounty = payoutPerJob;
const expectedAmountPerAnnotator =
BigInt(jobsPerAnnotator) *
ethers.parseUnits(trimmedBounty, tokenDecimals);
BigInt(jobsPerAnnotator) * expectedJobBounty;

const expectedPayouts = annotators.map((address) => ({
address,
Expand All @@ -168,7 +161,9 @@ describe('CvatPayoutsCalculator', () => {
];

const jobsPerAnnotator = faker.number.int({ min: 1, max: 3 });
const tokenDecimals = BigInt(faker.number.int({ min: 6, max: 18 }));
const reservedFunds = BigInt(
faker.number.int({ min: 1000, max: 100000 }).toString(),
);

const annotationsMeta: CvatAnnotationMeta = {
jobs: Array.from(
Expand Down Expand Up @@ -207,7 +202,7 @@ describe('CvatPayoutsCalculator', () => {
mockedStorageService.downloadJsonLikeData.mockResolvedValueOnce(
annotationsMeta,
);
mockedWeb3Service.getTokenDecimals.mockResolvedValueOnce(tokenDecimals);
mockedGetReservedFunds.mockResolvedValueOnce(reservedFunds);

const manifest = generateCvatManifest();

Expand All @@ -218,9 +213,14 @@ describe('CvatPayoutsCalculator', () => {
finalResultsUrl: faker.internet.url(),
});

const matchedJobCount = annotationsMeta.jobs.filter((job) =>
annotationsMeta.results.some(
(result) => result.id === job.final_result_id,
),
).length;
const expectedJobBounty = reservedFunds / BigInt(matchedJobCount);
const expectedAmountPerAnnotator =
BigInt(jobsPerAnnotator) *
ethers.parseUnits(manifest.job_bounty, tokenDecimals);
BigInt(jobsPerAnnotator) * expectedJobBounty;

const expectedPayouts = annotators.map((address) => ({
address,
Expand All @@ -231,5 +231,33 @@ describe('CvatPayoutsCalculator', () => {
_.sortBy(expectedPayouts, 'address'),
);
});

it('throws when there are no jobs with matching final results', async () => {
mockedStorageService.downloadJsonLikeData.mockResolvedValueOnce({
jobs: [
{
job_id: faker.number.int(),
final_result_id: 1,
},
],
results: [
{
id: 2,
job_id: faker.number.int(),
Comment thread
portuu3 marked this conversation as resolved.
annotator_wallet_address: faker.finance.ethereumAddress(),
annotation_quality: faker.number.float(),
},
],
} as CvatAnnotationMeta);

await expect(
calculator.calculate({
chainId,
escrowAddress,
manifest: generateCvatManifest(),
finalResultsUrl: faker.internet.url(),
}),
).rejects.toThrow('Invalid annotation meta');
});
});
});
63 changes: 19 additions & 44 deletions ...racle/server/src/modules/escrow-completion/payouts-calculation/cvat-payouts-calculator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { EscrowClient } from '@human-protocol/sdk';
import { Injectable } from '@nestjs/common';
import { ethers } from 'ethers';
import type { OverrideProperties } from 'type-fest';

import { CVAT_VALIDATION_META_FILENAME } from '@/common/constants';
Expand All @@ -27,7 +26,6 @@ export class CvatPayoutsCalculator implements EscrowPayoutsCalculator {
) {}

async calculate({
manifest,
chainId,
escrowAddress,
}: CalculateCvatPayoutsInput): Promise<CalculatedPayout[]> {
Expand All @@ -41,40 +39,34 @@ export class CvatPayoutsCalculator implements EscrowPayoutsCalculator {
await this.storageService.downloadJsonLikeData<CvatAnnotationMeta>(
`${intermediateResultsUrl}/${CVAT_VALIDATION_META_FILENAME}`,
);

if (!annotations.jobs.length || !annotations.results.length) {
throw new Error('Invalid annotation meta');
}

const tokenAddress = await escrowClient.getTokenAddress(escrowAddress);
const tokenDecimals = await this.web3Service.getTokenDecimals(
chainId,
tokenAddress,
);
const jobBountyValue = this.parseJobBounty(
manifest.job_bounty,
Number(tokenDecimals),
);
const workersBounties = new Map<string, bigint>();
const reservedFunds = await escrowClient.getReservedFunds(escrowAddress);

for (const job of annotations.jobs) {
const jobFinalResult = annotations.results.find(
(result) => result.id === job.final_result_id,
const matchedJobResults = annotations.jobs
.map((job) =>
annotations.results.find((result) => result.id === job.final_result_id),
)
.filter((result): result is CvatAnnotationMeta['results'][number] =>
Boolean(result),
);

if (!matchedJobResults.length) {
throw new Error('Invalid annotation meta');
}

const jobBountyValue = reservedFunds / BigInt(matchedJobResults.length);
const workersBounties = new Map<string, bigint>();

for (const jobFinalResult of matchedJobResults) {
// TODO: enable annotation quality validation when ready
if (
jobFinalResult
// && jobFinalResult.annotation_quality >= manifest.validation.min_quality
) {
const workerAddress = jobFinalResult.annotator_wallet_address;
const workerAddress = jobFinalResult.annotator_wallet_address;

const currentWorkerBounty = workersBounties.get(workerAddress) || 0n;
const currentWorkerBounty = workersBounties.get(workerAddress) || 0n;

workersBounties.set(
workerAddress,
currentWorkerBounty + jobBountyValue,
);
}
workersBounties.set(workerAddress, currentWorkerBounty + jobBountyValue);
}

return Array.from(workersBounties.entries()).map(
Expand All @@ -84,21 +76,4 @@ export class CvatPayoutsCalculator implements EscrowPayoutsCalculator {
}),
);
}

private parseJobBounty(jobBounty: string, tokenDecimals: number): bigint {
const parts = jobBounty.split('.');
if (parts.length > 1) {
const decimalsInBounty = parts[1].length;
if (decimalsInBounty > tokenDecimals) {
if (tokenDecimals === 0) {
return ethers.parseUnits(parts[0], tokenDecimals);
}
return ethers.parseUnits(
`${parts[0]}.${parts[1].slice(0, tokenDecimals)}`,
tokenDecimals,
);
}
}
return ethers.parseUnits(jobBounty, tokenDecimals);
}
}
7 changes: 3 additions & 4 deletions ...rver/src/modules/escrow-completion/payouts-calculation/default-payouts-calculator.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,7 @@ describe('DefaultPayoutsCalculator', () => {
});

describe('calculate', () => {
const reservedFunds = BigInt(faker.number.int({ min: 1000 }).toString());
const mockedGetReservedFunds = jest
.fn()
.mockImplementation(async () => reservedFunds);
const mockedGetReservedFunds = jest.fn().mockResolvedValue(0n);

beforeEach(() => {
mockedEscrowClient.build.mockResolvedValue({
Expand All @@ -74,7 +71,9 @@ describe('DefaultPayoutsCalculator', () => {
requestType: MarketingJobType.SOCIAL_MEDIA_PROMOTION,
submissionsRequired: faker.number.int({ min: 2, max: 5 }),
};
const reservedFunds = BigInt(faker.number.int({ min: 1000 }).toString());

mockedGetReservedFunds.mockResolvedValueOnce(reservedFunds);
mockedStorageService.downloadJsonLikeData.mockResolvedValueOnce(results);

const payouts = await calculator.calculate({
Expand Down
Loading