From db91549d3e67a3b1ecc924499654021e7b2f73ad Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Wed, 27 Sep 2023 12:30:24 +0330 Subject: [PATCH 1/3] Add Other types of campaigns to projectBySlug webservice https://github.com/Giveth/impact-graph/issues/1051#issuecomment-1733713799 --- config/example.env | 3 + config/test.env | 2 + src/entities/campaign.ts | 17 +++ src/repositories/projectRepository.test.ts | 47 +++++-- src/repositories/projectRepository.ts | 11 ++ src/resolvers/projectResolver.test.ts | 136 ++++++++++++++++++++- src/resolvers/projectResolver.ts | 25 ++-- src/services/campaignService.ts | 59 +++++++-- 8 files changed, 272 insertions(+), 28 deletions(-) diff --git a/config/example.env b/config/example.env index 86aa13b79..9ecd2a5f9 100644 --- a/config/example.env +++ b/config/example.env @@ -202,3 +202,6 @@ POWER_BALANCE_AGGREGATOR_ADAPTER=powerBalanceAggregator NUMBER_OF_BALANCE_AGGREGATOR_BATCH=20 QF_ROUND_ESTIMATED_MATCHING_CACHE_DURATION=60000 + +# OPTIONAL - default: Every 10 minutes +PROJECT_CAMPAIGNS_CACHE_DURATION=600000 diff --git a/config/test.env b/config/test.env index b9bb1c11c..a5786e08c 100644 --- a/config/test.env +++ b/config/test.env @@ -175,3 +175,5 @@ NUMBER_OF_BALANCE_AGGREGATOR_BATCH=7 # ! millisecond cache, if we increase cache in test ENV we might get some errors in tests QF_ROUND_ESTIMATED_MATCHING_CACHE_DURATION=1 +# ! millisecond cache, if we increase cache in test ENV we might get some errors in tests +PROJECT_CAMPAIGNS_CACHE_DURATION=1 diff --git a/src/entities/campaign.ts b/src/entities/campaign.ts index fcd3d8c21..7e1860e10 100644 --- a/src/entities/campaign.ts +++ b/src/entities/campaign.ts @@ -35,9 +35,26 @@ export enum CampaignFilterField { } export enum CampaignType { + // https://github.com/Giveth/impact-graph/blob/staging/docs/campaignsInstruction.md + + // In these type of projects we pick some projects to show them in campaign, + // for instance for Turkey earthquake we pick some projects. + // so we just need to add slug of those projects in Related Projects Slugs and in + // what order we add them they will be shown in frontend ManuallySelected = 'ManuallySelected', + + // Sometimes in a campaign we just want to show projects in an specified order, + // for instance we can create a campaign like ** Check projects that received most likes** so for + // this campaign you set SortField as campaign type and then you can use one of below sorting fields SortField = 'SortField', + + // Sometimes we need to filter some projects in a campaign, + // for instance Let's verified projects that accept funds on Gnosis chain, + // for this we can Add verified and acceptFundOnGnosis filters FilterFields = 'FilterFields', + + // Some campaigns don't include any project in them and they are just some banner + // like Feeling $nice? campaign in below image WithoutProjects = 'WithoutProjects', } diff --git a/src/repositories/projectRepository.test.ts b/src/repositories/projectRepository.test.ts index 50cce6ef4..b4a20ce89 100644 --- a/src/repositories/projectRepository.test.ts +++ b/src/repositories/projectRepository.test.ts @@ -1,6 +1,7 @@ import { findProjectById, findProjectBySlug, + findProjectBySlugWithoutAnyJoin, findProjectByWalletAddress, findProjectsByIdArray, findProjectsBySlugArray, @@ -56,6 +57,20 @@ describe( updateDescriptionSummaryTestCases, ); +describe('verifyProject test cases', verifyProjectTestCases); +describe('verifyMultipleProjects test cases', verifyMultipleProjectsTestCases); +describe('findProjectById test cases', findProjectByIdTestCases); +describe('findProjectsByIdArray test cases', findProjectsByIdArrayTestCases); +describe('findProjectBySlug test cases', findProjectBySlugTestCases); +describe( + 'findProjectBySlugWithoutAnyJoin test cases', + findProjectBySlugWithoutAnyJoinTestCases, +); +describe( + 'findProjectsBySlugArray test cases', + findProjectsBySlugArrayTestCases, +); + function projectsWithoutUpdateAfterTimeFrameTestCases() { it('should return projects created a long time ago', async () => { const superExpiredProject = await saveProjectDirectlyToDb({ @@ -96,22 +111,13 @@ function projectsWithoutUpdateAfterTimeFrameTestCases() { }); } -describe('verifyProject test cases', verifyProjectTestCases); -describe('verifyMultipleProjects test cases', verifyMultipleProjectsTestCases); -describe('findProjectById test cases', findProjectByIdTestCases); -describe('findProjectsByIdArray test cases', findProjectsByIdArrayTestCases); -describe('findProjectBySlug test cases', findProjectBySlugTestCases); -describe( - 'findProjectsBySlugArray test cases', - findProjectsBySlugArrayTestCases, -); - function findProjectBySlugTestCases() { - it('Should find project by id', async () => { + it('Should find project by slug', async () => { const project = await saveProjectDirectlyToDb(createProjectData()); const foundProject = await findProjectBySlug(project.slug as string); assert.isOk(foundProject); assert.equal(foundProject?.id, project.id); + assert.isOk(foundProject?.adminUser); }); it('should not find project when project doesnt exists', async () => { @@ -120,6 +126,25 @@ function findProjectBySlugTestCases() { }); } +function findProjectBySlugWithoutAnyJoinTestCases() { + it('Should find project by slug', async () => { + const project = await saveProjectDirectlyToDb(createProjectData()); + const foundProject = await findProjectBySlugWithoutAnyJoin( + project.slug as string, + ); + assert.isOk(foundProject); + assert.equal(foundProject?.id, project.id); + assert.isNotOk(foundProject?.adminUser); + }); + + it('should not find project when project doesnt exists', async () => { + const foundProject = await findProjectBySlugWithoutAnyJoin( + new Date().toString(), + ); + assert.isNull(foundProject); + }); +} + function findProjectsBySlugArrayTestCases() { it('Should find project multi projects by slug', async () => { const project1 = await saveProjectDirectlyToDb(createProjectData()); diff --git a/src/repositories/projectRepository.ts b/src/repositories/projectRepository.ts index 72f7003b4..25e762dce 100644 --- a/src/repositories/projectRepository.ts +++ b/src/repositories/projectRepository.ts @@ -211,6 +211,17 @@ export const findProjectBySlug = (slug: string): Promise => { ); }; +export const findProjectBySlugWithoutAnyJoin = ( + slug: string, +): Promise => { + // check current slug and previous slugs + return Project.createQueryBuilder('project') + .where(`:slug = ANY(project."slugHistory") or project.slug = :slug`, { + slug, + }) + .getOne(); +}; + export const verifyMultipleProjects = async (params: { verified: boolean; projectsIds: string[] | number[]; diff --git a/src/resolvers/projectResolver.test.ts b/src/resolvers/projectResolver.test.ts index 0aaae662f..882c4edad 100644 --- a/src/resolvers/projectResolver.test.ts +++ b/src/resolvers/projectResolver.test.ts @@ -90,7 +90,12 @@ import { refreshUserProjectPowerView } from '../repositories/userProjectPowerVie import { AppDataSource } from '../orm'; // We are using cache so redis needs to be cleared for tests with same filters import { redis } from '../redis'; -import { Campaign, CampaignType } from '../entities/campaign'; +import { + Campaign, + CampaignFilterField, + CampaignSortingField, + CampaignType, +} from '../entities/campaign'; import { generateRandomString, getHtmlTextSummary } from '../utils/utils'; import { FeaturedUpdate } from '../entities/featuredUpdate'; import { @@ -5456,7 +5461,7 @@ function projectBySlugTestCases() { assert.isTrue(project.projectPower.totalPower > 0); }); - it('should return projects including active campaigns', async () => { + it('should return projects including active ManuallySelected campaigns', async () => { const projectWithCampaign = await saveProjectDirectlyToDb({ ...createProjectData(), title: String(new Date().getTime()), @@ -5505,6 +5510,133 @@ function projectBySlugTestCases() { await campaign.remove(); }); + it('should return projects including active SortField campaigns', async () => { + const projectWithCampaign = await saveProjectDirectlyToDb({ + ...createProjectData(), + title: String(new Date().getTime()), + slug: String(new Date().getTime()), + }); + const campaign = await Campaign.create({ + isActive: true, + type: CampaignType.SortField, + sortingField: CampaignSortingField.Newest, + slug: generateRandomString(), + title: 'title1', + description: 'description1', + photo: 'https://google.com', + order: 1, + }).save(); + const result = await axios.post(graphqlUrl, { + query: fetchProjectsBySlugQuery, + variables: { + slug: projectWithCampaign.slug, + }, + }); + + const project = result.data.data.projectBySlug; + assert.equal(Number(project.id), projectWithCampaign.id); + + assert.exists(project.campaigns); + assert.isNotEmpty(project.campaigns); + assert.equal(project.campaigns[0].id, campaign.id); + + const projectWithoutCampaignResult = await axios.post(graphqlUrl, { + query: fetchProjectsBySlugQuery, + variables: { + // and old project that I'm sure it would not be in the Newest campaign + slug: SEED_DATA.FIRST_PROJECT.slug, + }, + }); + + const project2 = projectWithoutCampaignResult.data.data.projectBySlug; + assert.equal(Number(project2.id), SEED_DATA.FIRST_PROJECT.id); + + assert.isEmpty(project2.campaigns); + + await campaign.remove(); + }); + + it('should return projects including active FilterField campaigns (acceptOnGnosis)', async () => { + // In this filter the default sorting for projects is givPower so I need to create a project with power + // to be sure that it will be in the campaign + await PowerBoosting.clear(); + await InstantPowerBalance.clear(); + + const user1 = await saveUserDirectlyToDb(generateRandomEtheriumAddress()); + + const projectWithCampaign = await saveProjectDirectlyToDb({ + ...createProjectData(), + title: String(new Date().getTime()), + slug: String(new Date().getTime()), + networkId: NETWORK_IDS.XDAI, + }); + + const projectWithoutCampaign = await saveProjectDirectlyToDb({ + ...createProjectData(), + title: String(new Date().getTime()), + slug: String(new Date().getTime()), + networkId: NETWORK_IDS.POLYGON, + }); + + await Promise.all( + [[user1, projectWithCampaign, 10]].map(item => { + const [user, project, percentage] = item as [User, Project, number]; + return insertSinglePowerBoosting({ + user, + project, + percentage, + }); + }), + ); + + await saveOrUpdateInstantPowerBalances([ + { + userId: user1.id, + balance: 10000, + balanceAggregatorUpdatedAt: new Date(1_000_000), + }, + ]); + + await updateInstantBoosting(); + + const campaign = await Campaign.create({ + isActive: true, + type: CampaignType.FilterFields, + filterFields: [CampaignFilterField.acceptFundOnGnosis], + slug: generateRandomString(), + title: 'title1', + description: 'description1', + photo: 'https://google.com', + order: 1, + }).save(); + const result = await axios.post(graphqlUrl, { + query: fetchProjectsBySlugQuery, + variables: { + slug: projectWithCampaign.slug, + }, + }); + + const fetchedProject = result.data.data.projectBySlug; + assert.equal(Number(fetchedProject.id), projectWithCampaign.id); + + assert.exists(fetchedProject.campaigns); + assert.isNotEmpty(fetchedProject.campaigns); + assert.equal(fetchedProject.campaigns[0].id, campaign.id); + + const projectWithoutCampaignResult = await axios.post(graphqlUrl, { + query: fetchProjectsBySlugQuery, + variables: { + slug: projectWithoutCampaign.slug, + }, + }); + + const project2 = projectWithoutCampaignResult.data.data.projectBySlug; + assert.equal(Number(project2.id), projectWithoutCampaign.id); + + assert.isEmpty(project2.campaigns); + + await campaign.remove(); + }); it('should return projects including active campaigns, even when sent slug is in the slugHistory of project', async () => { const projectWithCampaign = await saveProjectDirectlyToDb({ diff --git a/src/resolvers/projectResolver.ts b/src/resolvers/projectResolver.ts index 2ec7ab8b2..6e246deff 100644 --- a/src/resolvers/projectResolver.ts +++ b/src/resolvers/projectResolver.ts @@ -77,6 +77,7 @@ import { FilterProjectQueryInputParams, filterProjectsQuery, findProjectById, + findProjectBySlugWithoutAnyJoin, totalProjectsPerDate, totalProjectsPerDateByMonthAndYear, userIsOwnerOfProject, @@ -108,6 +109,7 @@ import { FeaturedUpdate } from '../entities/featuredUpdate'; import { PROJECT_UPDATE_CONTENT_MAX_LENGTH } from '../constants/validators'; import { calculateGivbackFactor } from '../services/givbackService'; import { ProjectBySlugResponse } from './types/projectResolver'; +import { getAllProjectsRelatedToActiveCampaigns } from '../services/campaignService'; @ObjectType() class AllProjects { @@ -898,11 +900,18 @@ export class ProjectResolver { isOwnerOfProject = await userIsOwnerOfProject(viewerUserId, slug); } + const minimalProject = await findProjectBySlugWithoutAnyJoin(slug); + if (!minimalProject) { + throw new Error(i18n.__(translationErrorMessagesKeys.PROJECT_NOT_FOUND)); + } + const campaignSlugs = (await getAllProjectsRelatedToActiveCampaigns())[ + minimalProject.id + ]; + let query = this.projectRepository .createQueryBuilder('project') - // check current slug and previous slugs - .where(`:slug = ANY(project."slugHistory") or project.slug = :slug`, { - slug, + .where(`project.id = :id`, { + id: minimalProject.id, }) .leftJoinAndSelect('project.status', 'status') .leftJoinAndSelect( @@ -922,9 +931,10 @@ export class ProjectResolver { 'project.campaigns', Campaign, 'campaigns', - '(campaigns."relatedProjectsSlugs" && ARRAY[:slug]::text[] OR campaigns."relatedProjectsSlugs" && project."slugHistory") AND campaigns."isActive" = TRUE', + '((campaigns."relatedProjectsSlugs" && ARRAY[:slug]::text[] OR campaigns."relatedProjectsSlugs" && project."slugHistory") AND campaigns."isActive" = TRUE) OR (campaigns.slug = ANY(:campaignSlugs))', { slug, + campaignSlugs, }, ) .leftJoin('project.adminUser', 'user') @@ -946,9 +956,6 @@ export class ProjectResolver { }); const project = await query.getOne(); - if (!project) { - throw new Error(i18n.__(translationErrorMessagesKeys.PROJECT_NOT_FOUND)); - } canUserVisitProject(project, String(user?.userId)); const verificationForm = project?.projectVerificationForm || @@ -956,7 +963,9 @@ export class ProjectResolver { if (verificationForm) { (project as Project).verificationFormStatus = verificationForm?.status; } - const { givbackFactor } = await calculateGivbackFactor(project.id); + + // We know that we have the project because if we reach this line means minimalProject is not null + const { givbackFactor } = await calculateGivbackFactor(project!.id); return { ...project, givbackFactor }; } diff --git a/src/services/campaignService.ts b/src/services/campaignService.ts index 1851d13af..ede5e7265 100644 --- a/src/services/campaignService.ts +++ b/src/services/campaignService.ts @@ -7,16 +7,15 @@ import { FilterField, Project, SortingField } from '../entities/project'; import { findUserReactionsByProjectIds } from '../repositories/reactionRepository'; import { ModuleThread, Pool } from 'threads'; import { ProjectResolverWorker } from '../workers/projectsResolverWorker'; +import { QueryBuilder } from 'typeorm/query-builder/QueryBuilder'; +import { findAllActiveCampaigns } from '../repositories/campaignRepository'; const projectFiltersCacheDuration = Number(process.env.PROJECT_FILTERS_THREADS_POOL_DURATION) || 60000; -export const fillCampaignProjects = async (params: { - userId?: number; - campaign: Campaign; - projectsFiltersThreadPool: Pool>; -}): Promise => { - const { campaign, userId, projectsFiltersThreadPool } = params; +const createFetchCampaignProjectsQuery = ( + campaign: Campaign, +): FilterProjectQueryInputParams | null => { const limit = 10; const skip = 0; const projectsQueryParams: FilterProjectQueryInputParams = { @@ -36,9 +35,55 @@ export const fillCampaignProjects = async (params: { campaign.sortingField as unknown as SortingField; } else if (campaign.type === CampaignType.WithoutProjects) { // Dont add projects to this campaign type - return campaign; + return null; + } + + return projectsQueryParams; +}; +let projectCampaignCache: { [key: number]: string[] } | undefined; + +export const getAllProjectsRelatedToActiveCampaigns = async (): Promise<{ + [key: number]: string[]; +}> => { + // It returns all project and campaigns( excluding manuallySelectedCampaign) + if (projectCampaignCache) { + return projectCampaignCache; } + projectCampaignCache = {}; + const activeCampaigns = await findAllActiveCampaigns(); + for (const campaign of activeCampaigns) { + const projectsQueryParams = createFetchCampaignProjectsQuery(campaign); + if (!projectsQueryParams) { + break; + } + const projectsQuery = filterProjectsQuery(projectsQueryParams); + const projects = await projectsQuery.getMany(); + for (const project of projects) { + projectCampaignCache[project.id] + ? projectCampaignCache[project.id].push(campaign.slug) + : (projectCampaignCache[project.id] = [campaign.slug]); + } + } + const projectCampaignsCacheDuration = + Number(process.env.PROJECT_CAMPAIGNS_CACHE_DURATION) || 10 * 60 * 1000; + setTimeout(() => { + // We make it undefined every 10 minutes, to refresh it + projectCampaignCache = undefined; + }, projectCampaignsCacheDuration); + + return projectCampaignCache; +}; +export const fillCampaignProjects = async (params: { + userId?: number; + campaign: Campaign; + projectsFiltersThreadPool: Pool>; +}): Promise => { + const { campaign, userId, projectsFiltersThreadPool } = params; + const projectsQueryParams = createFetchCampaignProjectsQuery(campaign); + if (!projectsQueryParams) { + return campaign; + } const projectsQuery = filterProjectsQuery(projectsQueryParams); const projectsQueryCacheKey = await projectsFiltersThreadPool.queue(hasher => hasher.hashProjectFilters({ From 704cb987c0d4063ad75af9e446fbe2557edd62b5 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Thu, 28 Sep 2023 14:38:45 +0330 Subject: [PATCH 2/3] Change name of fetchProjectsBySlugQuery field --- src/resolvers/projectResolver.test.ts | 44 ++++++++++---------- src/resolvers/qfRoundHistoryResolver.test.ts | 2 +- test/graphqlQueries.ts | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/resolvers/projectResolver.test.ts b/src/resolvers/projectResolver.test.ts index 882c4edad..7ae005553 100644 --- a/src/resolvers/projectResolver.test.ts +++ b/src/resolvers/projectResolver.test.ts @@ -29,7 +29,7 @@ import { fetchLikedProjectsQuery, fetchMultiFilterAllProjectsQuery, fetchNewProjectsPerDate, - fetchProjectsBySlugQuery, + fetchProjectBySlugQuery, fetchProjectUpdatesQuery, fetchSimilarProjectsBySlugQuery, getProjectsAcceptTokensQuery, @@ -5324,7 +5324,7 @@ function projectBySlugTestCases() { const result = await axios.post( graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, connectedWalletUserId: user!.id, @@ -5367,7 +5367,7 @@ function projectBySlugTestCases() { }).save(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, connectedWalletUserId: user!.id, @@ -5390,7 +5390,7 @@ function projectBySlugTestCases() { }); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, @@ -5448,7 +5448,7 @@ function projectBySlugTestCases() { await refreshProjectPowerView(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, @@ -5484,7 +5484,7 @@ function projectBySlugTestCases() { order: 1, }).save(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: projectWithCampaign.slug, }, @@ -5497,7 +5497,7 @@ function projectBySlugTestCases() { assert.isNotEmpty(project.campaigns); const projectWithoutCampaignResult = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: projectWithoutCampaign.slug, }, @@ -5527,7 +5527,7 @@ function projectBySlugTestCases() { order: 1, }).save(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: projectWithCampaign.slug, }, @@ -5541,7 +5541,7 @@ function projectBySlugTestCases() { assert.equal(project.campaigns[0].id, campaign.id); const projectWithoutCampaignResult = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { // and old project that I'm sure it would not be in the Newest campaign slug: SEED_DATA.FIRST_PROJECT.slug, @@ -5610,7 +5610,7 @@ function projectBySlugTestCases() { order: 1, }).save(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: projectWithCampaign.slug, }, @@ -5624,7 +5624,7 @@ function projectBySlugTestCases() { assert.equal(fetchedProject.campaigns[0].id, campaign.id); const projectWithoutCampaignResult = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: projectWithoutCampaign.slug, }, @@ -5659,7 +5659,7 @@ function projectBySlugTestCases() { order: 1, }).save(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: previousSlug, }, @@ -5755,7 +5755,7 @@ function projectBySlugTestCases() { await refreshProjectFuturePowerView(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, @@ -5850,7 +5850,7 @@ function projectBySlugTestCases() { await refreshProjectFuturePowerView(false); let result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, @@ -5865,7 +5865,7 @@ function projectBySlugTestCases() { await refreshProjectFuturePowerView(true); result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, @@ -5884,7 +5884,7 @@ function projectBySlugTestCases() { }); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: draftedProject.slug, }, @@ -5908,7 +5908,7 @@ function projectBySlugTestCases() { const result = await axios.post( graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: draftedProject.slug, connectedWalletUserId: SEED_DATA.FIRST_USER.id, @@ -5939,7 +5939,7 @@ function projectBySlugTestCases() { const result = await axios.post( graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: draftedProject.slug, }, @@ -5967,7 +5967,7 @@ function projectBySlugTestCases() { }); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project.slug, }, @@ -5991,7 +5991,7 @@ function projectBySlugTestCases() { const result = await axios.post( graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project.slug, connectedWalletUserId: SEED_DATA.FIRST_USER.id, @@ -6022,7 +6022,7 @@ function projectBySlugTestCases() { const result = await axios.post( graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: cancelledProject.slug, }, @@ -6080,7 +6080,7 @@ function projectBySlugTestCases() { await updateInstantBoosting(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, diff --git a/src/resolvers/qfRoundHistoryResolver.test.ts b/src/resolvers/qfRoundHistoryResolver.test.ts index 3f4e9fa85..3fbf8b0a4 100644 --- a/src/resolvers/qfRoundHistoryResolver.test.ts +++ b/src/resolvers/qfRoundHistoryResolver.test.ts @@ -14,7 +14,7 @@ import moment from 'moment'; import { fillQfRoundHistory } from '../repositories/qfRoundHistoryRepository'; import axios from 'axios'; import { - fetchProjectsBySlugQuery, + fetchProjectBySlugQuery, getQfRoundHistoryQuery, } from '../../test/graphqlQueries'; diff --git a/test/graphqlQueries.ts b/test/graphqlQueries.ts index e041a1625..a6bf0808a 100644 --- a/test/graphqlQueries.ts +++ b/test/graphqlQueries.ts @@ -782,7 +782,7 @@ export const getQfRoundHistoryQuery = ` } `; -export const fetchProjectsBySlugQuery = ` +export const fetchProjectBySlugQuery = ` query ( $slug: String! ) { From 0325f7c7d674a8070af7d926630c5ee9792899a3 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Mon, 2 Oct 2023 17:54:26 +0330 Subject: [PATCH 3/3] Move caching project campaign slugs process to a separated worker related #1051 --- config/example.env | 3 ++ config/test.env | 4 ++ src/resolvers/projectResolver.test.ts | 3 ++ src/server/bootstrap.ts | 3 ++ src/services/campaignService.ts | 38 +++++++++---------- .../cronJobs/checkActiveStatusQfRounds.ts | 17 --------- .../updateProjectCampaignsCacheJob.ts | 37 ++++++++++++++++++ src/workers/cacheProjectCampaignsWorker.ts | 17 +++++++++ 8 files changed, 86 insertions(+), 36 deletions(-) create mode 100644 src/services/cronJobs/updateProjectCampaignsCacheJob.ts create mode 100644 src/workers/cacheProjectCampaignsWorker.ts diff --git a/config/example.env b/config/example.env index 9ecd2a5f9..9e61f4c03 100644 --- a/config/example.env +++ b/config/example.env @@ -205,3 +205,6 @@ QF_ROUND_ESTIMATED_MATCHING_CACHE_DURATION=60000 # OPTIONAL - default: Every 10 minutes PROJECT_CAMPAIGNS_CACHE_DURATION=600000 + +# OPTIONAL - default: */10 * * * * * ( Every 10 minutes) +PROJECT_CAMPAIGNS_CRONJOB_EXPRESSION=*/10 * * * * * diff --git a/config/test.env b/config/test.env index a5786e08c..71fce3f69 100644 --- a/config/test.env +++ b/config/test.env @@ -177,3 +177,7 @@ NUMBER_OF_BALANCE_AGGREGATOR_BATCH=7 QF_ROUND_ESTIMATED_MATCHING_CACHE_DURATION=1 # ! millisecond cache, if we increase cache in test ENV we might get some errors in tests PROJECT_CAMPAIGNS_CACHE_DURATION=1 + + +# OPTIONAL - default: */10 * * * * * ( Every 10 minutes) +PROJECT_CAMPAIGNS_CRONJOB_EXPRESSION=*/10 * * * * * diff --git a/src/resolvers/projectResolver.test.ts b/src/resolvers/projectResolver.test.ts index 7ae005553..84276636f 100644 --- a/src/resolvers/projectResolver.test.ts +++ b/src/resolvers/projectResolver.test.ts @@ -114,6 +114,7 @@ import { } from '../services/projectViewsService'; import { addOrUpdatePowerSnapshotBalances } from '../repositories/powerBalanceSnapshotRepository'; import { findPowerSnapshots } from '../repositories/powerSnapshotRepository'; +import { cacheProjectCampaigns } from '../services/campaignService'; const ARGUMENT_VALIDATION_ERROR_MESSAGE = new ArgumentValidationError([ { property: '' }, @@ -5526,6 +5527,7 @@ function projectBySlugTestCases() { photo: 'https://google.com', order: 1, }).save(); + await cacheProjectCampaigns(); const result = await axios.post(graphqlUrl, { query: fetchProjectBySlugQuery, variables: { @@ -5609,6 +5611,7 @@ function projectBySlugTestCases() { photo: 'https://google.com', order: 1, }).save(); + await cacheProjectCampaigns(); const result = await axios.post(graphqlUrl, { query: fetchProjectBySlugQuery, variables: { diff --git a/src/server/bootstrap.ts b/src/server/bootstrap.ts index f371922bc..51d498681 100644 --- a/src/server/bootstrap.ts +++ b/src/server/bootstrap.ts @@ -66,6 +66,8 @@ import { } from '../services/projectViewsService'; import { isTestEnv } from '../utils/utils'; import { runCheckActiveStatusOfQfRounds } from '../services/cronJobs/checkActiveStatusQfRounds'; +import { runUpdateProjectCampaignsCacheJob } from '../services/cronJobs/updateProjectCampaignsCacheJob'; +import { getAllProjectsRelatedToActiveCampaigns } from '../services/campaignService'; Resource.validate = validate; @@ -372,6 +374,7 @@ export async function bootstrap() { runInstantBoostingUpdateCronJob(); } await runCheckActiveStatusOfQfRounds(); + await runUpdateProjectCampaignsCacheJob(); } catch (err) { logger.error(err); } diff --git a/src/services/campaignService.ts b/src/services/campaignService.ts index ede5e7265..d3c31707d 100644 --- a/src/services/campaignService.ts +++ b/src/services/campaignService.ts @@ -9,6 +9,7 @@ import { ModuleThread, Pool } from 'threads'; import { ProjectResolverWorker } from '../workers/projectsResolverWorker'; import { QueryBuilder } from 'typeorm/query-builder/QueryBuilder'; import { findAllActiveCampaigns } from '../repositories/campaignRepository'; +import { logger } from '../utils/logger'; const projectFiltersCacheDuration = Number(process.env.PROJECT_FILTERS_THREADS_POOL_DURATION) || 60000; @@ -40,38 +41,37 @@ const createFetchCampaignProjectsQuery = ( return projectsQueryParams; }; -let projectCampaignCache: { [key: number]: string[] } | undefined; +let projectCampaignCache: { [key: number]: string[] } = {}; -export const getAllProjectsRelatedToActiveCampaigns = async (): Promise<{ +export const getAllProjectsRelatedToActiveCampaigns = (): { [key: number]: string[]; -}> => { +} => { // It returns all project and campaigns( excluding manuallySelectedCampaign) - if (projectCampaignCache) { - return projectCampaignCache; - } - projectCampaignCache = {}; + return projectCampaignCache; +}; + +export const cacheProjectCampaigns = async (): Promise => { + logger.debug('cacheProjectCampaigns() has been called'); + const newProjectCampaignCache = {}; const activeCampaigns = await findAllActiveCampaigns(); for (const campaign of activeCampaigns) { const projectsQueryParams = createFetchCampaignProjectsQuery(campaign); if (!projectsQueryParams) { - break; + continue; } const projectsQuery = filterProjectsQuery(projectsQueryParams); const projects = await projectsQuery.getMany(); for (const project of projects) { - projectCampaignCache[project.id] - ? projectCampaignCache[project.id].push(campaign.slug) - : (projectCampaignCache[project.id] = [campaign.slug]); + newProjectCampaignCache[project.id] + ? newProjectCampaignCache[project.id].push(campaign.slug) + : (newProjectCampaignCache[project.id] = [campaign.slug]); } } - const projectCampaignsCacheDuration = - Number(process.env.PROJECT_CAMPAIGNS_CACHE_DURATION) || 10 * 60 * 1000; - setTimeout(() => { - // We make it undefined every 10 minutes, to refresh it - projectCampaignCache = undefined; - }, projectCampaignsCacheDuration); - - return projectCampaignCache; + projectCampaignCache = newProjectCampaignCache; + logger.debug( + 'cacheProjectCampaigns() ended successfully, projectCampaignCache size ', + Object.keys(projectCampaignCache).length, + ); }; export const fillCampaignProjects = async (params: { diff --git a/src/services/cronJobs/checkActiveStatusQfRounds.ts b/src/services/cronJobs/checkActiveStatusQfRounds.ts index 584700b91..b9519bd1b 100644 --- a/src/services/cronJobs/checkActiveStatusQfRounds.ts +++ b/src/services/cronJobs/checkActiveStatusQfRounds.ts @@ -1,23 +1,6 @@ import config from '../../config'; import { logger } from '../../utils/logger'; import { schedule } from 'node-cron'; -import { - getPowerRound, - setPowerRound, -} from '../../repositories/powerRoundRepository'; -import { getRoundNumberByDate } from '../../utils/powerBoostingUtils'; -import { - refreshProjectPowerView, - refreshProjectFuturePowerView, - getBottomRank, -} from '../../repositories/projectPowerViewRepository'; -import { refreshUserProjectPowerView } from '../../repositories/userProjectPowerViewRepository'; -import { - copyProjectRanksToPreviousRoundRankTable, - deleteAllPreviousRoundRanks, - projectsThatTheirRanksHaveChanged, -} from '../../repositories/previousRoundRankRepository'; -import { getNotificationAdapter } from '../../adapters/adaptersFactory'; import { isTestEnv, sleep } from '../../utils/utils'; import { deactivateExpiredQfRounds, diff --git a/src/services/cronJobs/updateProjectCampaignsCacheJob.ts b/src/services/cronJobs/updateProjectCampaignsCacheJob.ts new file mode 100644 index 000000000..3797278ab --- /dev/null +++ b/src/services/cronJobs/updateProjectCampaignsCacheJob.ts @@ -0,0 +1,37 @@ +import config from '../../config'; +import { logger } from '../../utils/logger'; +import { schedule } from 'node-cron'; +import { isTestEnv } from '../../utils/utils'; +import { ModuleThread, Pool, spawn, Worker } from 'threads'; +import { CacheProjectCampaignsWorker } from '../../workers/cacheProjectCampaignsWorker'; + +// every 10 minutes +const cronJobTime = + (config.get('CACHE_PROJECT_CAMPAIGNS_CRONJOB_EXPRESSION') as string) || + '*/10 * * * *'; + +const projectsFiltersThreadPool: Pool< + ModuleThread +> = Pool( + () => spawn(new Worker('../../workers/cacheProjectCampaignsWorker')), // create the worker, +); +export const runUpdateProjectCampaignsCacheJob = () => { + // Run it first time to make sure it is cached + projectsFiltersThreadPool.queue(async worker => { + await worker.cacheSlugsOfCampaignProjects(); + }); + + logger.debug( + 'runUpdateProjectCampaignsCacheJob() has been called, cronJobTime', + cronJobTime, + ); + schedule(cronJobTime, async () => { + try { + projectsFiltersThreadPool.queue(async worker => { + await worker.cacheSlugsOfCampaignProjects(); + }); + } catch (e) { + logger.error('runUpdateProjectCampaignsCacheJob() error', e); + } + }); +}; diff --git a/src/workers/cacheProjectCampaignsWorker.ts b/src/workers/cacheProjectCampaignsWorker.ts new file mode 100644 index 000000000..87b3568a9 --- /dev/null +++ b/src/workers/cacheProjectCampaignsWorker.ts @@ -0,0 +1,17 @@ +// workers/auth.js +import { expose } from 'threads/worker'; +import { WorkerModule } from 'threads/dist/types/worker'; +import { cacheProjectCampaigns } from '../services/campaignService'; + +type ProjectsResolverWorkerFunctions = 'cacheSlugsOfCampaignProjects'; + +export type CacheProjectCampaignsWorker = + WorkerModule; + +const worker: CacheProjectCampaignsWorker = { + async cacheSlugsOfCampaignProjects() { + await cacheProjectCampaigns(); + }, +}; + +expose(worker);