Skip to content

Commit

Permalink
Merge pull request #1136 from Giveth/f_1051_fix_add_campaigns_to_proj…
Browse files Browse the repository at this point in the history
…ect_by_slug

Add Other types of campaigns to projectBySlug webservice
  • Loading branch information
mohammadranjbarz authored Oct 3, 2023
2 parents 86fea20 + 0325f7c commit 05eeae0
Show file tree
Hide file tree
Showing 10 changed files with 110 additions and 60 deletions.
3 changes: 3 additions & 0 deletions config/example.env
Original file line number Diff line number Diff line change
Expand Up @@ -205,3 +205,6 @@ QF_ROUND_ESTIMATED_MATCHING_CACHE_DURATION=60000

# OPTIONAL - default: Every 10 minutes
PROJECT_CAMPAIGNS_CACHE_DURATION=600000

# OPTIONAL - default: */10 * * * * * ( Every 10 minutes)
PROJECT_CAMPAIGNS_CRONJOB_EXPRESSION=*/10 * * * * *
4 changes: 4 additions & 0 deletions config/test.env
Original file line number Diff line number Diff line change
Expand Up @@ -177,3 +177,7 @@ NUMBER_OF_BALANCE_AGGREGATOR_BATCH=7
QF_ROUND_ESTIMATED_MATCHING_CACHE_DURATION=1
# ! millisecond cache, if we increase cache in test ENV we might get some errors in tests
PROJECT_CAMPAIGNS_CACHE_DURATION=1


# OPTIONAL - default: */10 * * * * * ( Every 10 minutes)
PROJECT_CAMPAIGNS_CRONJOB_EXPRESSION=*/10 * * * * *
47 changes: 25 additions & 22 deletions src/resolvers/projectResolver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import {
fetchLikedProjectsQuery,
fetchMultiFilterAllProjectsQuery,
fetchNewProjectsPerDate,
fetchProjectsBySlugQuery,
fetchProjectBySlugQuery,
fetchProjectUpdatesQuery,
fetchSimilarProjectsBySlugQuery,
getProjectsAcceptTokensQuery,
Expand Down Expand Up @@ -114,6 +114,7 @@ import {
} from '../services/projectViewsService';
import { addOrUpdatePowerSnapshotBalances } from '../repositories/powerBalanceSnapshotRepository';
import { findPowerSnapshots } from '../repositories/powerSnapshotRepository';
import { cacheProjectCampaigns } from '../services/campaignService';

const ARGUMENT_VALIDATION_ERROR_MESSAGE = new ArgumentValidationError([
{ property: '' },
Expand Down Expand Up @@ -5324,7 +5325,7 @@ function projectBySlugTestCases() {
const result = await axios.post(
graphqlUrl,
{
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: project1.slug,
connectedWalletUserId: user!.id,
Expand Down Expand Up @@ -5367,7 +5368,7 @@ function projectBySlugTestCases() {
}).save();

const result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: project1.slug,
connectedWalletUserId: user!.id,
Expand All @@ -5390,7 +5391,7 @@ function projectBySlugTestCases() {
});

const result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: project1.slug,
},
Expand Down Expand Up @@ -5448,7 +5449,7 @@ function projectBySlugTestCases() {

await refreshProjectPowerView();
const result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: project1.slug,
},
Expand Down Expand Up @@ -5484,7 +5485,7 @@ function projectBySlugTestCases() {
order: 1,
}).save();
const result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: projectWithCampaign.slug,
},
Expand All @@ -5497,7 +5498,7 @@ function projectBySlugTestCases() {
assert.isNotEmpty(project.campaigns);

const projectWithoutCampaignResult = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: projectWithoutCampaign.slug,
},
Expand Down Expand Up @@ -5526,8 +5527,9 @@ function projectBySlugTestCases() {
photo: 'https://google.com',
order: 1,
}).save();
await cacheProjectCampaigns();
const result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: projectWithCampaign.slug,
},
Expand All @@ -5541,7 +5543,7 @@ function projectBySlugTestCases() {
assert.equal(project.campaigns[0].id, campaign.id);

const projectWithoutCampaignResult = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
// and old project that I'm sure it would not be in the Newest campaign
slug: SEED_DATA.FIRST_PROJECT.slug,
Expand Down Expand Up @@ -5609,8 +5611,9 @@ function projectBySlugTestCases() {
photo: 'https://google.com',
order: 1,
}).save();
await cacheProjectCampaigns();
const result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: projectWithCampaign.slug,
},
Expand All @@ -5624,7 +5627,7 @@ function projectBySlugTestCases() {
assert.equal(fetchedProject.campaigns[0].id, campaign.id);

const projectWithoutCampaignResult = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: projectWithoutCampaign.slug,
},
Expand Down Expand Up @@ -5659,7 +5662,7 @@ function projectBySlugTestCases() {
order: 1,
}).save();
const result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: previousSlug,
},
Expand Down Expand Up @@ -5755,7 +5758,7 @@ function projectBySlugTestCases() {
await refreshProjectFuturePowerView();

const result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: project1.slug,
},
Expand Down Expand Up @@ -5850,7 +5853,7 @@ function projectBySlugTestCases() {
await refreshProjectFuturePowerView(false);

let result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: project1.slug,
},
Expand All @@ -5865,7 +5868,7 @@ function projectBySlugTestCases() {
await refreshProjectFuturePowerView(true);

result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: project1.slug,
},
Expand All @@ -5884,7 +5887,7 @@ function projectBySlugTestCases() {
});

const result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: draftedProject.slug,
},
Expand All @@ -5908,7 +5911,7 @@ function projectBySlugTestCases() {
const result = await axios.post(
graphqlUrl,
{
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: draftedProject.slug,
connectedWalletUserId: SEED_DATA.FIRST_USER.id,
Expand Down Expand Up @@ -5939,7 +5942,7 @@ function projectBySlugTestCases() {
const result = await axios.post(
graphqlUrl,
{
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: draftedProject.slug,
},
Expand Down Expand Up @@ -5967,7 +5970,7 @@ function projectBySlugTestCases() {
});

const result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: project.slug,
},
Expand All @@ -5991,7 +5994,7 @@ function projectBySlugTestCases() {
const result = await axios.post(
graphqlUrl,
{
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: project.slug,
connectedWalletUserId: SEED_DATA.FIRST_USER.id,
Expand Down Expand Up @@ -6022,7 +6025,7 @@ function projectBySlugTestCases() {
const result = await axios.post(
graphqlUrl,
{
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: cancelledProject.slug,
},
Expand Down Expand Up @@ -6080,7 +6083,7 @@ function projectBySlugTestCases() {
await updateInstantBoosting();

const result = await axios.post(graphqlUrl, {
query: fetchProjectsBySlugQuery,
query: fetchProjectBySlugQuery,
variables: {
slug: project1.slug,
},
Expand Down
2 changes: 1 addition & 1 deletion src/resolvers/qfRoundHistoryResolver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import moment from 'moment';
import { fillQfRoundHistory } from '../repositories/qfRoundHistoryRepository';
import axios from 'axios';
import {
fetchProjectsBySlugQuery,
fetchProjectBySlugQuery,
getQfRoundHistoryQuery,
} from '../../test/graphqlQueries';

Expand Down
3 changes: 3 additions & 0 deletions src/server/bootstrap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ import {
} from '../services/projectViewsService';
import { isTestEnv } from '../utils/utils';
import { runCheckActiveStatusOfQfRounds } from '../services/cronJobs/checkActiveStatusQfRounds';
import { runUpdateProjectCampaignsCacheJob } from '../services/cronJobs/updateProjectCampaignsCacheJob';
import { getAllProjectsRelatedToActiveCampaigns } from '../services/campaignService';

Resource.validate = validate;

Expand Down Expand Up @@ -372,6 +374,7 @@ export async function bootstrap() {
runInstantBoostingUpdateCronJob();
}
await runCheckActiveStatusOfQfRounds();
await runUpdateProjectCampaignsCacheJob();
} catch (err) {
logger.error(err);
}
Expand Down
38 changes: 19 additions & 19 deletions src/services/campaignService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { ModuleThread, Pool } from 'threads';
import { ProjectResolverWorker } from '../workers/projectsResolverWorker';
import { QueryBuilder } from 'typeorm/query-builder/QueryBuilder';
import { findAllActiveCampaigns } from '../repositories/campaignRepository';
import { logger } from '../utils/logger';

const projectFiltersCacheDuration =
Number(process.env.PROJECT_FILTERS_THREADS_POOL_DURATION) || 60000;
Expand Down Expand Up @@ -40,38 +41,37 @@ const createFetchCampaignProjectsQuery = (

return projectsQueryParams;
};
let projectCampaignCache: { [key: number]: string[] } | undefined;
let projectCampaignCache: { [key: number]: string[] } = {};

export const getAllProjectsRelatedToActiveCampaigns = async (): Promise<{
export const getAllProjectsRelatedToActiveCampaigns = (): {
[key: number]: string[];
}> => {
} => {
// It returns all project and campaigns( excluding manuallySelectedCampaign)
if (projectCampaignCache) {
return projectCampaignCache;
}
projectCampaignCache = {};
return projectCampaignCache;
};

export const cacheProjectCampaigns = async (): Promise<void> => {
logger.debug('cacheProjectCampaigns() has been called');
const newProjectCampaignCache = {};
const activeCampaigns = await findAllActiveCampaigns();
for (const campaign of activeCampaigns) {
const projectsQueryParams = createFetchCampaignProjectsQuery(campaign);
if (!projectsQueryParams) {
break;
continue;
}
const projectsQuery = filterProjectsQuery(projectsQueryParams);
const projects = await projectsQuery.getMany();
for (const project of projects) {
projectCampaignCache[project.id]
? projectCampaignCache[project.id].push(campaign.slug)
: (projectCampaignCache[project.id] = [campaign.slug]);
newProjectCampaignCache[project.id]
? newProjectCampaignCache[project.id].push(campaign.slug)
: (newProjectCampaignCache[project.id] = [campaign.slug]);
}
}
const projectCampaignsCacheDuration =
Number(process.env.PROJECT_CAMPAIGNS_CACHE_DURATION) || 10 * 60 * 1000;
setTimeout(() => {
// We make it undefined every 10 minutes, to refresh it
projectCampaignCache = undefined;
}, projectCampaignsCacheDuration);

return projectCampaignCache;
projectCampaignCache = newProjectCampaignCache;
logger.debug(
'cacheProjectCampaigns() ended successfully, projectCampaignCache size ',
Object.keys(projectCampaignCache).length,
);
};

export const fillCampaignProjects = async (params: {
Expand Down
17 changes: 0 additions & 17 deletions src/services/cronJobs/checkActiveStatusQfRounds.ts
Original file line number Diff line number Diff line change
@@ -1,23 +1,6 @@
import config from '../../config';
import { logger } from '../../utils/logger';
import { schedule } from 'node-cron';
import {
getPowerRound,
setPowerRound,
} from '../../repositories/powerRoundRepository';
import { getRoundNumberByDate } from '../../utils/powerBoostingUtils';
import {
refreshProjectPowerView,
refreshProjectFuturePowerView,
getBottomRank,
} from '../../repositories/projectPowerViewRepository';
import { refreshUserProjectPowerView } from '../../repositories/userProjectPowerViewRepository';
import {
copyProjectRanksToPreviousRoundRankTable,
deleteAllPreviousRoundRanks,
projectsThatTheirRanksHaveChanged,
} from '../../repositories/previousRoundRankRepository';
import { getNotificationAdapter } from '../../adapters/adaptersFactory';
import { isTestEnv, sleep } from '../../utils/utils';
import {
deactivateExpiredQfRounds,
Expand Down
37 changes: 37 additions & 0 deletions src/services/cronJobs/updateProjectCampaignsCacheJob.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import config from '../../config';
import { logger } from '../../utils/logger';
import { schedule } from 'node-cron';
import { isTestEnv } from '../../utils/utils';
import { ModuleThread, Pool, spawn, Worker } from 'threads';
import { CacheProjectCampaignsWorker } from '../../workers/cacheProjectCampaignsWorker';

// every 10 minutes
const cronJobTime =
(config.get('CACHE_PROJECT_CAMPAIGNS_CRONJOB_EXPRESSION') as string) ||
'*/10 * * * *';

const projectsFiltersThreadPool: Pool<
ModuleThread<CacheProjectCampaignsWorker>
> = Pool(
() => spawn(new Worker('../../workers/cacheProjectCampaignsWorker')), // create the worker,
);
export const runUpdateProjectCampaignsCacheJob = () => {
// Run it first time to make sure it is cached
projectsFiltersThreadPool.queue(async worker => {
await worker.cacheSlugsOfCampaignProjects();
});

logger.debug(
'runUpdateProjectCampaignsCacheJob() has been called, cronJobTime',
cronJobTime,
);
schedule(cronJobTime, async () => {
try {
projectsFiltersThreadPool.queue(async worker => {
await worker.cacheSlugsOfCampaignProjects();
});
} catch (e) {
logger.error('runUpdateProjectCampaignsCacheJob() error', e);
}
});
};
17 changes: 17 additions & 0 deletions src/workers/cacheProjectCampaignsWorker.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
// workers/auth.js
import { expose } from 'threads/worker';
import { WorkerModule } from 'threads/dist/types/worker';
import { cacheProjectCampaigns } from '../services/campaignService';

type ProjectsResolverWorkerFunctions = 'cacheSlugsOfCampaignProjects';

export type CacheProjectCampaignsWorker =
WorkerModule<ProjectsResolverWorkerFunctions>;

const worker: CacheProjectCampaignsWorker = {
async cacheSlugsOfCampaignProjects() {
await cacheProjectCampaigns();
},
};

expose(worker);
Loading

0 comments on commit 05eeae0

Please sign in to comment.