From 5c6010fc247ae6a762592d6817b65a66244f8b0a Mon Sep 17 00:00:00 2001 From: icemedia001 Date: Sun, 1 Sep 2024 14:21:10 +0100 Subject: [PATCH 1/7] Updated the posthog branch with required changes --- src/App/app.module.ts | 19 ++++++ src/Relayer/services/relayer.service.ts | 90 ++++++++++++++++--------- 2 files changed, 76 insertions(+), 33 deletions(-) diff --git a/src/App/app.module.ts b/src/App/app.module.ts index 9f6e3ffd..f3d1191b 100644 --- a/src/App/app.module.ts +++ b/src/App/app.module.ts @@ -66,6 +66,7 @@ import TagRepository from './Tag/tag.repository' import EventsService from './Wiki/events.service' import AppService from './app.service' import WikiController from './Wiki/controllers/wiki.controller' +import { PosthogModule } from 'nestjs-posthog' // istanbul ignore next @Module({ @@ -93,6 +94,24 @@ import WikiController from './Wiki/controllers/wiki.controller' ], }, }), + PosthogModule.forRootAsync({ + imports: [ConfigModule], + inject: [ConfigService], + useFactory: (configService: ConfigService) => { + const apiKey = configService.get('POSTHOG_API_KEY') + const host = configService.get('POSTHOG_API_URL') + if (!apiKey ||!host) { + console.error('Posthog configuration is missing apiKey or host') + } + return { + apiKey: apiKey || '', + options: { + host, + }, + mock: true, + } + }, + }), SitemapModule, MailerModule, httpModule(20000), diff --git a/src/Relayer/services/relayer.service.ts b/src/Relayer/services/relayer.service.ts index fb18d110..60263910 100644 --- a/src/Relayer/services/relayer.service.ts +++ b/src/Relayer/services/relayer.service.ts @@ -12,6 +12,7 @@ import WikiAbi from '../utils/wiki.abi' import USER_ACTIVITY_LIMIT from '../../globalVars' import ActivityRepository from '../../App/Activities/activity.repository' import AppService from '../../App/app.service' +import { PosthogService } from 'nestjs-posthog' @Injectable() class RelayerService { @@ -24,6 +25,7 @@ class RelayerService { private configService: ConfigService, private httpService: HttpService, private activityRepository: ActivityRepository, + private posthogService: PosthogService, ) { this.signer = this.getRelayerInstance() this.wikiInstance = this.getWikiContractInstance(this.signer) @@ -120,40 +122,62 @@ class RelayerService { } let result - if (this.appService.apiLevel() !== 'prod') { - const txConfig = { - gasPrice: ethers.utils.parseUnits('0.7', 'gwei'), + try { + if (this.appService.apiLevel() !== 'prod') { + const txConfig = { + gasPrice: ethers.utils.parseUnits('0.7', 'gwei'), + } + result = await this.wikiInstance.postBySig( + ipfs, + userAddr, + deadline, + v, + r, + s, + txConfig, + ) + } else { + const gas = await this.getUpdatedGas() + + const txConfig = this.appService.privateSigner() + ? { + gasPrice: ethers.utils.parseUnits(gas, 'gwei'), + gasLimit: 50000, + } + : { + gasLimit: 50000, + } + + result = await this.wikiInstance.postBySig( + ipfs, + userAddr, + deadline, + v, + r, + s, + txConfig, + ) } - result = await this.wikiInstance.postBySig( - ipfs, - userAddr, - deadline, - v, - r, - s, - txConfig, - ) - } else { - const gas = await this.getUpdatedGas() - - const txConfig = this.appService.privateSigner() - ? { - gasPrice: ethers.utils.parseUnits(gas, 'gwei'), - gasLimit: 50000, - } - : { - gasLimit: 50000, - } - - result = await this.wikiInstance.postBySig( - ipfs, - userAddr, - deadline, - v, - r, - s, - txConfig, - ) + this.posthogService.capture({ + distinctId: userAddr, + event: 'Successful Transaction', + properties: { + ipfs, + userAddr, + result, + }, + }) + } catch (error) { + this.posthogService.capture({ + distinctId: userAddr, + event: 'Failed Transaction', + properties: { + ipfs, + userAddr, + error, + }, + }) + throw error } return result } From 786e59f20b730c54426f209343f53e85389aeab5 Mon Sep 17 00:00:00 2001 From: icemedia001 Date: Mon, 2 Sep 2024 00:19:41 +0100 Subject: [PATCH 2/7] Lint --- package.json | 1 + pnpm-lock.yaml | 45 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 2c29b289..531fa793 100644 --- a/package.json +++ b/package.json @@ -93,6 +93,7 @@ "pg": "^8.7.3", "pg-boss": "^8.1.1", "pm2": "^5.2.2", + "posthog-js": "^1.160.0", "reflect-metadata": "^0.1.13", "remove-markdown": "^0.5.0", "rimraf": "^6.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 235288cc..2a999c86 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -188,6 +188,9 @@ importers: pm2: specifier: ^5.2.2 version: 5.2.2(bufferutil@4.0.8)(utf-8-validate@5.0.10) + posthog-js: + specifier: ^1.160.0 + version: 1.160.0 reflect-metadata: specifier: ^0.1.13 version: 0.1.13 @@ -3714,6 +3717,9 @@ packages: fecha@4.2.3: resolution: {integrity: sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==} + fflate@0.4.8: + resolution: {integrity: sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==} + figures@3.2.0: resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==} engines: {node: '>=8'} @@ -5719,11 +5725,17 @@ packages: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} + posthog-js@1.160.0: + resolution: {integrity: sha512-K/RRgmPYIpP69nnveCJfkclb8VU+R+jsgqlrKaLGsM5CtQM9g01WOzAiT3u36WLswi58JiFMXgJtECKQuoqTgQ==} + posthog-node@1.3.0: resolution: {integrity: sha512-2+VhqiY/rKIqKIXyvemBFHbeijHE25sP7eKltnqcFqAssUE6+sX6vusN9A4luzToOqHQkUZexiCKxvuGagh7JA==} engines: {node: '>=4'} hasBin: true + preact@10.23.2: + resolution: {integrity: sha512-kKYfePf9rzKnxOAKDpsWhg/ysrHPqT+yQ7UW4JjdnqjFIeNUnNcEJvhuA8fDenxAGWzUqtd51DfVg7xp/8T9NA==} + prelude-ls@1.1.2: resolution: {integrity: sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==} engines: {node: '>= 0.8.0'} @@ -6947,6 +6959,9 @@ packages: resolution: {integrity: sha512-AIihwH+ZmdHfkJm7BjSXiEClVt4zUFqX4YlFAzjL13wLtDuUneSaFvDBTbdYRecs35SiU7iNKbMnN+++wVfb6A==} engines: {node: '>=10.0.0'} + web-vitals@4.2.3: + resolution: {integrity: sha512-/CFAm1mNxSmOj6i0Co+iGFJ58OS4NRGVP+AWS/l509uIK5a1bSoIVaHz/ZumpHTfHSZBpgrJ+wjfpAOrTHok5Q==} + web3-core-helpers@1.10.4: resolution: {integrity: sha512-r+L5ylA17JlD1vwS8rjhWr0qg7zVoVMDvWhajWA5r5+USdh91jRUYosp19Kd1m2vE034v7Dfqe1xYRoH2zvG0g==} engines: {node: '>=8.0.0'} @@ -8998,7 +9013,7 @@ snapshots: '@pm2/pm2-version-check@1.0.4': dependencies: - debug: 4.3.4(supports-color@9.2.1) + debug: 4.3.6 transitivePeerDependencies: - supports-color @@ -9718,7 +9733,7 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.3.4(supports-color@9.2.1) + debug: 4.3.6 transitivePeerDependencies: - supports-color @@ -9992,7 +10007,7 @@ snapshots: avvio@7.2.5: dependencies: archy: 1.0.0 - debug: 4.3.4(supports-color@9.2.1) + debug: 4.3.6 fastq: 1.13.0 queue-microtask: 1.2.3 transitivePeerDependencies: @@ -11551,6 +11566,8 @@ snapshots: fecha@4.2.3: {} + fflate@0.4.8: {} + figures@3.2.0: dependencies: escape-string-regexp: 1.0.5 @@ -11791,7 +11808,7 @@ snapshots: dependencies: '@tootallnate/once': 1.1.2 data-uri-to-buffer: 3.0.1 - debug: 4.3.4(supports-color@9.2.1) + debug: 4.3.6 file-uri-to-path: 2.0.0 fs-extra: 8.1.0 ftp: 0.3.10 @@ -12072,7 +12089,7 @@ snapshots: dependencies: '@tootallnate/once': 1.1.2 agent-base: 6.0.2 - debug: 4.3.4(supports-color@9.2.1) + debug: 4.3.6 transitivePeerDependencies: - supports-color @@ -13906,7 +13923,7 @@ snapshots: dependencies: '@tootallnate/once': 1.1.2 agent-base: 6.0.2 - debug: 4.3.4(supports-color@9.2.1) + debug: 4.3.6 get-uri: 3.0.2 http-proxy-agent: 4.0.1 https-proxy-agent: 5.0.1 @@ -14177,6 +14194,12 @@ snapshots: dependencies: xtend: 4.0.2 + posthog-js@1.160.0: + dependencies: + fflate: 0.4.8 + preact: 10.23.2 + web-vitals: 4.2.3 + posthog-node@1.3.0: dependencies: axios: 0.24.0 @@ -14190,6 +14213,8 @@ snapshots: transitivePeerDependencies: - debug + preact@10.23.2: {} + prelude-ls@1.1.2: {} prelude-ls@1.2.1: {} @@ -14249,7 +14274,7 @@ snapshots: proxy-agent@5.0.0: dependencies: agent-base: 6.0.2 - debug: 4.3.4(supports-color@9.2.1) + debug: 4.3.6 http-proxy-agent: 4.0.1 https-proxy-agent: 5.0.1 lru-cache: 5.1.1 @@ -14469,7 +14494,7 @@ snapshots: require-in-the-middle@5.2.0: dependencies: - debug: 4.3.4(supports-color@9.2.1) + debug: 4.3.6 module-details-from-path: 1.0.3 resolve: 1.22.1 transitivePeerDependencies: @@ -14782,7 +14807,7 @@ snapshots: socks-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.4(supports-color@9.2.1) + debug: 4.3.6 socks: 2.7.1 transitivePeerDependencies: - supports-color @@ -15429,6 +15454,8 @@ snapshots: transitivePeerDependencies: - encoding + web-vitals@4.2.3: {} + web3-core-helpers@1.10.4: dependencies: web3-eth-iban: 1.10.4 From 982089c8bc82a8bca1431b7d023610be5bf56fa3 Mon Sep 17 00:00:00 2001 From: icemedia001 Date: Mon, 2 Sep 2024 00:39:22 +0100 Subject: [PATCH 3/7] Temporary fix --- src/App/marketCap/marketCap.service.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/App/marketCap/marketCap.service.ts b/src/App/marketCap/marketCap.service.ts index 42bbe3a5..754cafab 100644 --- a/src/App/marketCap/marketCap.service.ts +++ b/src/App/marketCap/marketCap.service.ts @@ -358,7 +358,6 @@ class MarketCapService { category: string, id?: string, ): Promise { - return null const wikiRepository = this.dataSource.getRepository(Wiki) const baseCoingeckoUrl = 'https://www.coingecko.com/en' const coingeckoProfileUrl = `${baseCoingeckoUrl}/${ @@ -389,7 +388,7 @@ class MarketCapService { } const wiki = await queryBuilder.getOne() - return wiki + return wiki || null } } From 569fa4f57407622503f9f0a25e3b6b851cb8c679 Mon Sep 17 00:00:00 2001 From: icemedia001 Date: Mon, 2 Sep 2024 01:20:11 +0100 Subject: [PATCH 4/7] Added missing module --- src/Relayer/relayer.module.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Relayer/relayer.module.ts b/src/Relayer/relayer.module.ts index fd372996..3352b017 100644 --- a/src/Relayer/relayer.module.ts +++ b/src/Relayer/relayer.module.ts @@ -5,9 +5,10 @@ import RelayerResolver from './resolvers/relayer.resolver' import httpModule from '../httpModule' import ActivityModule from '../App/Activities/activity.module' import AppService from '../App/app.service' +import { POSTHOG_MODULE_OPTIONS, PosthogModule } from 'nestjs-posthog' @Module({ - imports: [httpModule(10000), ActivityModule], + imports: [httpModule(10000), ActivityModule, PosthogModule], controllers: [RelayerController], providers: [RelayerService, RelayerResolver, AppService], }) From 1bf4361136e8dfff84307f916e2a94d3f7279474 Mon Sep 17 00:00:00 2001 From: icemedia001 Date: Sun, 22 Sep 2024 20:18:55 +0100 Subject: [PATCH 5/7] Merge main --- .codeclimate.yml | 17 + package.json | 1 + src/App/Blog/blog.dto.ts | 98 ++++++ src/App/Blog/blog.module.ts | 13 + src/App/Blog/blog.resolver.ts | 22 ++ src/App/Blog/blog.service.ts | 222 +++++++++++++ src/App/Blog/mirrorApi.service.ts | 117 +++++++ .../Subscriptions/subscriptions.service.ts | 6 +- src/App/Wiki/events.service.ts | 34 +- src/App/Wiki/wiki.resolver.ts | 56 +++- src/App/Wiki/wiki.service.ts | 165 ++++++--- src/App/Wiki/wikiStats.dto.ts | 12 +- src/App/app.module.ts | 12 +- src/App/marketCap/marketCap.resolver.ts | 2 +- src/App/marketCap/marketCap.service.ts | 313 +++++++++--------- .../revalidatePage/revalidatePage.service.ts | 64 +++- src/App/utils/discordWebhookHandler.ts | 2 +- src/App/utils/test-helpers/testHelpers.ts | 2 +- src/Database/Entities/explorer.entity.ts | 27 ++ src/Database/database.module.ts | 2 + .../indexerWebhook.controller.spec.ts | 2 +- src/Relayer/services/relayer.service.ts | 2 +- src/posthog/posthog.module.ts | 2 +- 23 files changed, 934 insertions(+), 259 deletions(-) create mode 100644 src/App/Blog/blog.dto.ts create mode 100644 src/App/Blog/blog.module.ts create mode 100644 src/App/Blog/blog.resolver.ts create mode 100644 src/App/Blog/blog.service.ts create mode 100644 src/App/Blog/mirrorApi.service.ts create mode 100644 src/Database/Entities/explorer.entity.ts diff --git a/.codeclimate.yml b/.codeclimate.yml index 65573947..966df63c 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -24,3 +24,20 @@ checks: return-statements: config: threshold: 4 +plugins: + duplication: + enabled: true + config: + languages: + typescript: + patterns: + - "**/*.ts" + exclude_patterns: + - "**/*.d.ts" + mass_threshold: 65 + filters: + - "(.*?)@(Module|Controller|Injectable|Catch|ExceptionFilter|Middleware|NestMiddleware|Pipe|UsePipes|UseGuards|UseInterceptors|UseFilters)\\(.*?\\)" + - "(.*?)@(Get|Post|Put|Delete|Patch|Options|Head|All)\\(.*?\\)" + - "(.*?)@(Param|Query|Body|Headers|Req|Res)\\(.*?\\)" + - "(.*?)@(ObjectType|InputType|Headers|Req|Res)\\(.*?\\)" + - "(.*?)@Field\\((.*?) => (.*?)\\)" \ No newline at end of file diff --git a/package.json b/package.json index 531fa793..ea5eedfb 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "@types/remove-markdown": "^0.3.1", "apollo-server-express": "^3.13.0", "apollo-server-fastify": "^3.13.0", + "arweave": "^1.15.1", "axios": "^1.3.1", "cache-manager": "^3.6.1", "class-transformer": "^0.5.1", diff --git a/src/App/Blog/blog.dto.ts b/src/App/Blog/blog.dto.ts new file mode 100644 index 00000000..fde83c96 --- /dev/null +++ b/src/App/Blog/blog.dto.ts @@ -0,0 +1,98 @@ +/* eslint-disable max-classes-per-file */ +import { ObjectType, Field, Int } from '@nestjs/graphql' + +@ObjectType() +export class Address { + @Field() + address!: string +} + +@ObjectType() +export class Project { + @Field() + project!: Address +} + +@ObjectType() +export class FormatedBlogType { + @Field() + title!: string + + @Field() + slug!: string + + @Field({ nullable: true }) + body?: string + + @Field({ nullable: true }) + excerpt?: string + + @Field() + digest!: string + + @Field() + contributor!: string + + @Field({ nullable: true }) + timestamp?: number + + @Field({ nullable: true }) + cover_image?: string + + @Field() + image_sizes!: number +} + +@ObjectType() +export class Blog extends FormatedBlogType { + @Field({ nullable: true }) + publishedAtTimestamp?: number + + @Field({ nullable: true }) + transaction?: string + + @Field({ nullable: true }) + publisher?: Project +} + +@ObjectType() +export class EntryPathPicked { + @Field() + slug!: string + + @Field({ nullable: true }) + timestamp?: number +} + +@ObjectType() +export class EntryPath extends EntryPathPicked { + @Field() + path!: string +} + +@ObjectType() +export class BlogTag { + @Field() + name!: string + + @Field() + value!: string +} + +@ObjectType() +export class Block { + @Field(() => Int) + timestamp!: number +} + +@ObjectType() +export class BlogPostType { + @Field({ nullable: true }) + maxW?: string + + @Field(() => Blog) + post!: Blog + + @Field(() => Int) + key!: number +} diff --git a/src/App/Blog/blog.module.ts b/src/App/Blog/blog.module.ts new file mode 100644 index 00000000..e0d235fd --- /dev/null +++ b/src/App/Blog/blog.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common' +import { ConfigModule } from '@nestjs/config' +import { HttpModule } from '@nestjs/axios' +import BlogService from './blog.service' +import BlogResolver from './blog.resolver' +import MirrorApiService from './mirrorApi.service' + +@Module({ + imports: [ConfigModule, HttpModule], + providers: [BlogService, BlogResolver, MirrorApiService], + exports: [BlogService], +}) +export default class BlogModule {} diff --git a/src/App/Blog/blog.resolver.ts b/src/App/Blog/blog.resolver.ts new file mode 100644 index 00000000..b58e8f86 --- /dev/null +++ b/src/App/Blog/blog.resolver.ts @@ -0,0 +1,22 @@ +import { Resolver, Query, Args } from '@nestjs/graphql' +import BlogService from './blog.service' +import { Blog } from './blog.dto' + +@Resolver(() => Blog) +class BlogResolver { + constructor(private readonly blogService: BlogService) {} + + @Query(() => [Blog], { nullable: 'items' }) + async getBlogs(): Promise { + return this.blogService.getBlogsFromAccounts() + } + + @Query(() => Blog, { nullable: true }) + async getBlog( + @Args('digest', { type: () => String }) digest: string, + ): Promise { + return this.blogService.getBlogByDigest(digest) + } +} + +export default BlogResolver diff --git a/src/App/Blog/blog.service.ts b/src/App/Blog/blog.service.ts new file mode 100644 index 00000000..d953df39 --- /dev/null +++ b/src/App/Blog/blog.service.ts @@ -0,0 +1,222 @@ +import { Injectable, Inject, CACHE_MANAGER } from '@nestjs/common' +import { Cache } from 'cache-manager' +import { ConfigService } from '@nestjs/config' +import Arweave from 'arweave' +import slugify from 'slugify' +import { Blog, BlogTag, EntryPath, FormatedBlogType } from './blog.dto' +import MirrorApiService from './mirrorApi.service' + +const arweave = Arweave.init({ + host: 'arweave.net', + protocol: 'https', + port: 443, + timeout: 5000, +}) + +export type BlogNode = { + id: string + block: { + timestamp: number + } + tags: BlogTag[] +} + +export type RawTransactions = { + transactions: { + edges: { + node: BlogNode + }[] + } +} + +@Injectable() +class BlogService { + private EVERIPEDIA_BLOG_ACCOUNT2: string + + private EVERIPEDIA_BLOG_ACCOUNT3: string + + private BLOG_CACHE_KEY = 'blog-cache' + + constructor( + @Inject(CACHE_MANAGER) private cacheManager: Cache, + private configService: ConfigService, + private readonly mirrorApiService: MirrorApiService, + ) { + this.EVERIPEDIA_BLOG_ACCOUNT2 = this.configService.get( + 'EVERIPEDIA_BLOG_ACCOUNT2', + ) as string + this.EVERIPEDIA_BLOG_ACCOUNT3 = this.configService.get( + 'EVERIPEDIA_BLOG_ACCOUNT3', + ) as string + } + + async formatEntry( + blog: Partial, + transactionId: string, + timestamp: number, + ): Promise { + return { + title: blog.title || '', + slug: slugify(blog.title || ''), + body: blog.body || '', + digest: blog.digest || '', + contributor: blog.contributor || '', + transaction: transactionId, + timestamp, + cover_image: blog.body + ? (blog.body + .split('\n\n')[0] + .match(/!\[[^\]]*\]\((.*?)\s*("(?:.*[^"])")?\s*\)/m) || [])?.[1] || + '' + : undefined, + image_sizes: 50, + } + } + + formatBlog(blog: Blog, hasBody?: boolean) { + const regex = /\*\*(.*?)\*\*/g + const newBlog: FormatedBlogType = { + title: blog?.title || '', + slug: slugify(blog?.title || ''), + digest: blog?.digest || '', + contributor: blog?.publisher?.project?.address || '', + timestamp: blog?.timestamp, + cover_image: blog?.body + ? (blog?.body + .split('\n\n')[0] + .match(/!\[[^\]]*\]\((.*?)\s*("(?:.*[^"])")?\s*\)/m) || [])?.[1] + : '', + image_sizes: 50, + } + if (hasBody) { + newBlog.body = blog?.body + newBlog.excerpt = blog?.body + ? blog?.body.split('\n\n')[1].replace(regex, '$1') + : '' + } + return newBlog + } + + async getBlogsFromAccounts(): Promise { + const accounts = [ + this.EVERIPEDIA_BLOG_ACCOUNT2, + this.EVERIPEDIA_BLOG_ACCOUNT3, + ] + + let blogs = await this.cacheManager.get(this.BLOG_CACHE_KEY) + if (!blogs) { + blogs = await Promise.all( + accounts.map(async (account) => { + try { + if (!account) return [] + const entries = await this.mirrorApiService.getBlogs(account) + return ( + entries + ?.filter((entry) => entry.publishedAtTimestamp) + .map((b: Blog) => this.formatBlog(b, true)) || [] + ) + } catch (error) { + console.log(`Error fetching blogs for account ${account}:`, error) + return [] + } + }), + ).then((blogArrays) => blogArrays.flat()) + } + await this.cacheManager.set(this.BLOG_CACHE_KEY, blogs, { ttl: 7200 }) + return blogs as Blog[] + } + + async getBlogByDigest(digest: string): Promise { + const blogs = await this.cacheManager.get(this.BLOG_CACHE_KEY) + + const blog = + blogs?.find((e: Blog) => e.digest === digest) ?? + (await this.mirrorApiService.getBlog(digest)) + + return this.formatBlog(blog as Blog, true) + } + + async getEntryPaths({ transactions }: RawTransactions): Promise { + return transactions.edges + .map(({ node }) => { + const tags = Object.fromEntries( + node.tags.map((tag: BlogTag) => [tag.name, tag.value]), + ) + + if (!node || !node.block) return { slug: '', path: '', timestamp: 0 } + + return { + slug: tags['Original-Content-Digest'], + path: node.id, + timestamp: node.block.timestamp, + } + }) + .filter((entry) => entry.slug && entry.slug !== '') + .reduce((acc: EntryPath[], current) => { + const x = acc.findIndex( + (entry: EntryPath) => entry.slug === current.slug, + ) + if (x === -1) return acc.concat([current]) + + acc[x].timestamp = current.timestamp + + return acc + }, []) + } + + async mapEntry(entry: EntryPath) { + try { + const result = await arweave.transactions.getData(entry.path, { + decode: true, + string: true, + }) + + const parsedResult = JSON.parse(result as string) + const { + content: { title, body }, + digest, + authorship: { contributor }, + transactionId, + } = parsedResult + + if (result) { + const formattedEntry = await this.formatEntry( + { title, body, digest, contributor, transaction: transactionId }, + entry.slug, + entry.timestamp || 0, + ) + + if (!formattedEntry.cover_image) { + return null + } + + return formattedEntry + } + return undefined + } catch (_error) { + console.error(`Error mapping entry: ${entry.path}`, _error) + return null + } + } + + async getBlogEntriesFormatted(entryPaths: EntryPath[]): Promise { + const entries = await Promise.all( + entryPaths.map(async (entry: EntryPath) => this.mapEntry(entry)), + ) + return entries + .sort((a, b) => { + if (a && b) return (b.timestamp || 0) - (a.timestamp || 0) + + return 0 + }) + .reduce((acc: Blog[], current) => { + const x = acc.find( + (entry: Blog) => current && entry.slug === current.slug, + ) + if (!x && current) return acc.concat([current as Blog]) + return acc + }, []) + } +} + +export default BlogService diff --git a/src/App/Blog/mirrorApi.service.ts b/src/App/Blog/mirrorApi.service.ts new file mode 100644 index 00000000..d12c01da --- /dev/null +++ b/src/App/Blog/mirrorApi.service.ts @@ -0,0 +1,117 @@ +import { HttpService } from '@nestjs/axios' +import { Injectable } from '@nestjs/common' +import { ConfigService } from '@nestjs/config' +import { gql } from 'graphql-request' +import { Blog } from './blog.dto' + +@Injectable() +class MirrorApiService { + private readonly apiUrl: string + + private readonly origin: string + + constructor( + private readonly httpService: HttpService, + private readonly configService: ConfigService, + ) { + this.apiUrl = + this.configService.get('MIRROR_API_URL') || + 'https://mirror-api.com/graphql' + this.origin = + this.configService.get('MIRROR_API_ORIGIN') || + 'https://mirror.xyz' + } + + private async executeQuery( + query: string, + variables: Record, + ): Promise { + const headers = { Origin: this.origin } + const response = await this.httpService + .post(this.apiUrl, { query, variables }, { headers }) + .toPromise() + + return response?.data?.data || {} + } + + async getBlogs(address: string): Promise { + const query = gql` + query Entries($projectAddress: String!) { + entries(projectAddress: $projectAddress) { + ...entryDetails + } + } + + fragment entryDetails on entry { + _id + body + digest + timestamp + title + publishedAtTimestamp + arweaveTransactionRequest { + transactionId + } + publisher { + ...publisherDetails + } + } + + fragment publisherDetails on PublisherType { + project { + ...projectDetails + } + } + + fragment projectDetails on ProjectType { + _id + address + } + ` + + const result = await this.executeQuery<{ entries: Blog[] }>(query, { + projectAddress: address, + }) + return result.entries || [] + } + + async getBlog(digest: string): Promise { + const query = gql` + query EntryWritingNFT($digest: String!) { + entry(digest: $digest) { + ...entryDetails + } + } + + fragment entryDetails on entry { + _id + body + digest + timestamp + title + arweaveTransactionRequest { + transactionId + } + publisher { + ...publisherDetails + } + } + + fragment publisherDetails on PublisherType { + project { + ...projectDetails + } + } + + fragment projectDetails on ProjectType { + _id + address + } + ` + + const result = await this.executeQuery<{ entry: Blog }>(query, { digest }) + return result.entry || null + } +} + +export default MirrorApiService diff --git a/src/App/Subscriptions/subscriptions.service.ts b/src/App/Subscriptions/subscriptions.service.ts index fdbead35..abb7b6eb 100644 --- a/src/App/Subscriptions/subscriptions.service.ts +++ b/src/App/Subscriptions/subscriptions.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@nestjs/common' -import { DataSource, Repository } from 'typeorm' +import { DataSource, ILike, Repository } from 'typeorm' import IqSubscription from '../../Database/Entities/IqSubscription' import { WikiSubscriptionArgs } from '../../Database/Entities/types/IWiki' import TokenValidator from '../utils/validateToken' @@ -20,7 +20,7 @@ class WikiSubscriptionService { ): Promise { return (await this.repository()).findOne({ where: { - userId: args.userId, + userId: ILike(args.userId), subscriptionType: args.subscriptionType, auxiliaryId: args.auxiliaryId, }, @@ -36,7 +36,7 @@ class WikiSubscriptionService { } return (await this.repository()).find({ where: { - userId: id, + userId: ILike(id), }, }) } diff --git a/src/App/Wiki/events.service.ts b/src/App/Wiki/events.service.ts index 206d2e83..107bfbaf 100644 --- a/src/App/Wiki/events.service.ts +++ b/src/App/Wiki/events.service.ts @@ -22,29 +22,18 @@ class EventsService { const queryBuilder = repository .createQueryBuilder('wiki') .innerJoin('wiki.tags', 'tag') - .leftJoinAndSelect('wiki.wikiEvents', 'wikiEvents') - .where('subWiki.hidden = :hidden', { hidden: false }) - .limit(args.limit) - .offset(args.offset) + .leftJoin('wiki.wikiEvents', 'wikiEvents') + .where('wiki.hidden = :hidden', { hidden: false }) + .groupBy('wiki.id') if (ids.length > 1) { - queryBuilder.where((qb) => { - const subQuery = qb - .subQuery() - .select('subWiki.id') - .from(Wiki, 'subWiki') - .innerJoin('subWiki.tags', 'subTag') - .where('LOWER(subTag.id) = ANY(:tagIds)', { - tagIds: ids.map((tag) => tag.toLowerCase()), - }) - .andWhere('subWiki.hidden = :hidden', { hidden: false }) - .groupBy('subWiki.id') - .having('COUNT(DISTINCT subTag.id) > 1') - .getQuery() - return `wiki.id IN ${subQuery}` - }) + queryBuilder + .andWhere('LOWER(tag.id) IN (:...ids)', { + ids: ids.map((id) => id.toLowerCase()), + }) + .having('COUNT(DISTINCT tag."id") > 1') } else { - queryBuilder.where('LOWER(tag.id) = LOWER(:ev)', { ev: eventTag }) + queryBuilder.andWhere('LOWER(tag.id) = LOWER(:ev)', { ev: eventTag }) } this.wikiService.applyDateFilter( @@ -55,6 +44,9 @@ class EventsService { switch (args.order) { case 'date': this.wikiService.eventDateOrder(queryBuilder, args.direction) + queryBuilder + .addGroupBy('wikiEvents.date') + .addGroupBy('wikiEvents.multiDateStart') break case 'id': @@ -66,7 +58,7 @@ class EventsService { break } - return await queryBuilder.getMany() + return await queryBuilder.limit(args.limit).offset(args.offset).getMany() } catch (error) { console.error('Error fetching events:', error) throw error diff --git a/src/App/Wiki/wiki.resolver.ts b/src/App/Wiki/wiki.resolver.ts index 55a18e14..4942e86f 100644 --- a/src/App/Wiki/wiki.resolver.ts +++ b/src/App/Wiki/wiki.resolver.ts @@ -33,6 +33,8 @@ import PageviewsPerDay from '../../Database/Entities/pageviewsPerPage.entity' import { PageViewArgs, VistArgs } from '../pageViews/pageviews.dto' import { updateDates } from '../utils/queryHelpers' import { eventWiki } from '../Tag/tag.dto' +import Explorer from '../../Database/Entities/explorer.entity' +import PaginationArgs from '../pagination.args' @UseInterceptors(AdminLogsInterceptor) @Resolver(() => Wiki) @@ -55,8 +57,12 @@ class WikiResolver { } @Query(() => [Wiki]) - async promotedWikis(@Args() args: LangArgs) { - return this.wikiService.getPromotedWikis(args) + async promotedWikis( + @Args() args: LangArgs, + @Args('featuredEvents', { type: () => Boolean, defaultValue: false }) + featuredEvents: boolean, + ) { + return this.wikiService.getPromotedWikis(args, featuredEvents) } @Query(() => [Wiki]) @@ -81,8 +87,12 @@ class WikiResolver { @Query(() => [Wiki]) @UseGuards(AuthGuard) - async wikisHidden(@Args() args: LangArgs) { - return this.wikiService.getWikisHidden(args) + async wikisHidden( + @Args() args: LangArgs, + @Args('featuredEvents', { type: () => Boolean, defaultValue: false }) + featuredEvents: boolean, + ) { + return this.wikiService.getWikisHidden(args, featuredEvents) } @Query(() => [WikiUrl]) @@ -102,6 +112,28 @@ class WikiResolver { return this.wikiService.getCategoryTotal(args) } + @Query(() => [Explorer]) + async searchExplorers(@Args() args: ByIdArgs) { + return this.wikiService.searchExplorers(args.id) + } + + @Query(() => [Explorer]) + async explorers(@Args() args: PaginationArgs) { + return this.wikiService.getExplorers(args) + } + + @Mutation(() => Explorer, { nullable: true }) + @UseGuards(AuthGuard) + async addExplorer(@Args() args: Explorer) { + return this.wikiService.addExplorer(args) + } + + @Mutation(() => Boolean, { nullable: true }) + @UseGuards(AuthGuard) + async updateExplorer(@Args() args: Explorer) { + return this.wikiService.updateExplorer(args) + } + @Mutation(() => Wiki, { nullable: true }) @UseGuards(AuthGuard) async promoteWiki(@Args() args: PromoteWikiArgs, @Context() ctx: any) { @@ -116,18 +148,23 @@ class WikiResolver { @Mutation(() => Wiki, { nullable: true }) @UseGuards(AuthGuard) - async hideWiki(@Args() args: ByIdArgs, @Context() ctx: any) { + async hideWiki( + @Args() args: ByIdArgs, + @Context() ctx: any, + @Args('featuredEvents', { type: () => Boolean, defaultValue: false }) + featuredEvents: boolean, + ) { const cacheId = ctx.req.ip + args.id - const wiki = await this.wikiService.hideWiki(args) - + const wiki = await this.wikiService.hideWiki(args, featuredEvents) + const tags = (await wiki?.tags) || [] if (wiki) { await this.revalidate.revalidatePage( RevalidateEndpoints.HIDE_WIKI, undefined, wiki.id, wiki.promoted, - eventWiki(wiki.tags), + eventWiki(tags), ) this.eventEmitter.emit('admin.action', `${cacheId}`) } @@ -140,6 +177,7 @@ class WikiResolver { const cacheId = ctx.req.ip + args.id const wiki = await this.wikiService.unhideWiki(args) + const tags = (await wiki?.tags) || [] if (wiki) { await this.revalidate.revalidatePage( @@ -147,7 +185,7 @@ class WikiResolver { undefined, wiki.id, wiki.promoted, - eventWiki(wiki.tags), + eventWiki(tags), ) this.eventEmitter.emit('admin.action', `${cacheId}`) } diff --git a/src/App/Wiki/wiki.service.ts b/src/App/Wiki/wiki.service.ts index 497bb59e..f790ab6b 100644 --- a/src/App/Wiki/wiki.service.ts +++ b/src/App/Wiki/wiki.service.ts @@ -4,6 +4,7 @@ import { ConfigService } from '@nestjs/config' import { Brackets, DataSource, Repository, SelectQueryBuilder } from 'typeorm' import { Cache } from 'cache-manager' import { HttpService } from '@nestjs/axios' +import slugify from 'slugify' import Wiki from '../../Database/Entities/wiki.entity' import { orderWikis, updateDates } from '../utils/queryHelpers' import { ValidSlug, Valid, Slug } from '../utils/validSlug' @@ -22,6 +23,8 @@ import { DateArgs, Count } from './wikiStats.dto' import { OrderBy, Direction } from '../general.args' import { PageViewArgs } from '../pageViews/pageviews.dto' import DiscordWebhookService from '../utils/discordWebhookService' +import Explorer from '../../Database/Entities/explorer.entity' +import PaginationArgs from '../pagination.args' @Injectable() class WikiService { @@ -110,13 +113,6 @@ class WikiService { ): Promise { const queryBuilder = (await this.repository()).createQueryBuilder('wiki') - const wikis: Wiki[] | undefined = await this.cacheManager.get( - 'promotedWikis', - ) - if (wikis) { - return wikis - } - queryBuilder .where('wiki.languageId = :lang', { lang: args.lang }) .andWhere('wiki.promoted > 0') @@ -128,11 +124,6 @@ class WikiService { this.filterFeaturedEvents(queryBuilder, featuredEvents) const promotedWikis = await queryBuilder.getMany() - - this.cacheManager.set('promotedWikis', promotedWikis, { - ttl: 3600, - }) - return promotedWikis } @@ -385,18 +376,73 @@ class WikiService { return this.validSlug.validateSlug(slugs[0]?.id) } - async getWikisHidden(args: LangArgs): Promise { - return (await this.repository()).find({ - where: { - language: { id: args.lang }, - hidden: true, - }, - take: args.limit, - skip: args.offset, - order: { - updated: 'DESC', - }, + async getWikisHidden( + args: LangArgs, + featuredEvents = false, + ): Promise { + const queryBuilder = (await this.repository()).createQueryBuilder('wiki') + + queryBuilder + .where('wiki.languageId = :lang', { lang: args.lang }) + .andWhere('wiki.hidden = true') + .orderBy('wiki.updated', 'DESC') + .skip(args.offset) + .take(args.limit) + + this.filterFeaturedEvents(queryBuilder, featuredEvents) + + const hiddenWikis = await queryBuilder.getMany() + + return hiddenWikis + } + + async searchExplorers(explorer: string) { + const repo = this.dataSource.manager.getRepository(Explorer) + return repo + .createQueryBuilder('explorer') + .where('LOWER(explorer.id) LIKE LOWER(:id)', { id: `%${explorer}%` }) + .getMany() + } + + async getExplorers(args: PaginationArgs) { + const repo = this.dataSource.manager.getRepository(Explorer) + return repo + .createQueryBuilder('explorer') + .skip(args.offset) + .take(args.limit) + .getMany() + } + + async addExplorer(args: Explorer) { + const repo = this.dataSource.manager.getRepository(Explorer) + const existExplorer = await repo.findOneBy({ ...args }) + if (!args.baseUrl.startsWith('https') || existExplorer) { + return null + } + const slugId = slugify(args.explorer, { + lower: true, + strict: true, + remove: /[*+~.()'"!:@]/g, }) + + const newExplorer = repo.create({ + ...args, + id: slugId, + }) + await repo.save(newExplorer) + return newExplorer + } + + async updateExplorer(args: Explorer) { + const repo = this.dataSource.manager.getRepository(Explorer) + const existExplorer = await repo.findOneBy({ id: args.id }) + + if (!existExplorer) { + return false + } + + await repo.update({ id: args.id }, args) + return true } async getAddressToWiki(address: string): Promise { @@ -442,42 +488,45 @@ class WikiService { const queryBuilder = (await this.repository()).createQueryBuilder('wiki') - if (level <= 10) { - if (level > 0) { - queryBuilder - .andWhere('wiki.promoted = :level', { level }) - .andWhere('wiki.hidden = false') - - this.filterFeaturedEvents(queryBuilder, featuredEvents) - } - const promotedWiki = await queryBuilder.getOne() + if (level > 0) { + queryBuilder + .andWhere('wiki.promoted = :level', { level }) + .andWhere('wiki.hidden = false') - if (promotedWiki) { - await ( - await this.repository() - ) - .createQueryBuilder() - .update(Wiki) - .set({ promoted: 0 }) - .where('id = :id', { id: promotedWiki.id }) - .execute() - } + this.filterFeaturedEvents(queryBuilder, featuredEvents) + } + const promotedWiki = await queryBuilder.getOne() + if (promotedWiki) { await ( await this.repository() ) .createQueryBuilder() .update(Wiki) - .set({ promoted: level }) - .where('id = :id', { id }) + .set({ promoted: 0 }) + .where('id = :id', { id: promotedWiki.id }) .execute() - return wiki } - return null + + await ( + await this.repository() + ) + .createQueryBuilder() + .update(Wiki) + .set({ promoted: level }) + .where('id = :id', { id }) + .execute() + + await this.reOrderPromotedwikis(featuredEvents) + + return wiki } - async hideWiki(args: ByIdArgs): Promise { - const wiki = (await this.repository()).findOneBy({ id: args.id }) + async hideWiki( + args: ByIdArgs, + featuredEvents: boolean, + ): Promise { + const wiki = await (await this.repository()).findOneBy({ id: args.id }) await ( await this.repository() ) @@ -487,10 +536,18 @@ class WikiService { .where('id = :id', { id: args.id }) .execute() - const currentPromotions = await this.getPromotedWikis({ - id: 'en', - direction: 'ASC', - } as unknown as LangArgs) + await this.reOrderPromotedwikis(featuredEvents) + return wiki + } + + async reOrderPromotedwikis(featuredEvents: boolean) { + const currentPromotions = await this.getPromotedWikis( + { + lang: 'en', + direction: 'ASC', + } as unknown as LangArgs, + featuredEvents, + ) if (currentPromotions.length > 0) { for (let index = 0; index < currentPromotions.length; index += 1) { @@ -504,12 +561,10 @@ class WikiService { .execute() } } - - return wiki } async unhideWiki(args: ByIdArgs): Promise { - const wiki = (await this.repository()).findOneBy({ id: args.id }) + const wiki = await (await this.repository()).findOneBy({ id: args.id }) await ( await this.repository() ) diff --git a/src/App/Wiki/wikiStats.dto.ts b/src/App/Wiki/wikiStats.dto.ts index 61c12da6..dd0485eb 100644 --- a/src/App/Wiki/wikiStats.dto.ts +++ b/src/App/Wiki/wikiStats.dto.ts @@ -28,10 +28,18 @@ export class WikiUserStats extends Count { @ArgsType() export class DateArgs { @Field(() => Int) - startDate = Math.round(new Date().setDate(new Date().getDate() - 7) / 1000) + startDate: number @Field(() => Int) - endDate = Math.round(Date.now() / 1000) + endDate: number + + constructor() { + const currentDate = new Date() + this.startDate = Math.round( + currentDate.setDate(currentDate.getDate() - 7) / 1000, + ) + this.endDate = Math.round(Date.now() / 1000) + } } @ArgsType() diff --git a/src/App/app.module.ts b/src/App/app.module.ts index f3d1191b..f0c07c21 100644 --- a/src/App/app.module.ts +++ b/src/App/app.module.ts @@ -14,6 +14,7 @@ import { EventEmitterModule } from '@nestjs/event-emitter' import { GraphqlInterceptor } from '@ntegral/nestjs-sentry' import { InMemoryLRUCache } from '@apollo/utils.keyvaluecache' import { APP_INTERCEPTOR } from '@nestjs/core' +import { PosthogModule } from 'nestjs-posthog' import WikiResolver from './Wiki/wiki.resolver' import LanguageResolver from './Language/language.resolver' import CategoryResolver from './Category/category.resolver' @@ -66,7 +67,10 @@ import TagRepository from './Tag/tag.repository' import EventsService from './Wiki/events.service' import AppService from './app.service' import WikiController from './Wiki/controllers/wiki.controller' -import { PosthogModule } from 'nestjs-posthog' +import BlogService from './Blog/blog.service' +import BlogModule from './Blog/blog.module' +import BlogResolver from './Blog/blog.resolver' +import MirrorApiService from './Blog/mirrorApi.service' // istanbul ignore next @Module({ @@ -74,7 +78,7 @@ import { PosthogModule } from 'nestjs-posthog' ConfigModule.forRoot({ isGlobal: true, }), - CacheModule.register({ ttl: 3600 }), + CacheModule.register({ ttl: 3600, max: 10000, isGlobal: true }), GraphQLModule.forRoot({ driver: ApolloDriver, debug: true, @@ -128,6 +132,7 @@ import { PosthogModule } from 'nestjs-posthog' HiIQHolderModule, IQHolderModule, DiscordModule, + BlogModule, SentryMod, ], controllers: [UploadController, WikiController], @@ -162,8 +167,11 @@ import { PosthogModule } from 'nestjs-posthog' MarketCapResolver, MarketCapService, SentryPlugin, + BlogService, EventsResolver, EventsService, + BlogResolver, + MirrorApiService, { provide: APP_INTERCEPTOR, useFactory: () => diff --git a/src/App/marketCap/marketCap.resolver.ts b/src/App/marketCap/marketCap.resolver.ts index 38db2f15..76b463b2 100644 --- a/src/App/marketCap/marketCap.resolver.ts +++ b/src/App/marketCap/marketCap.resolver.ts @@ -23,7 +23,7 @@ class MarketCapResolver { @Query(() => [MarketRankData], { nullable: 'items' }) async rankList( @Args() args: MarketCapInputs, - ): Promise { + ): Promise<(NftRankListData | TokenRankListData)[]> { return this.marketCapService.ranks(args) } diff --git a/src/App/marketCap/marketCap.service.ts b/src/App/marketCap/marketCap.service.ts index 754cafab..2ffb7602 100644 --- a/src/App/marketCap/marketCap.service.ts +++ b/src/App/marketCap/marketCap.service.ts @@ -1,7 +1,5 @@ -/* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable no-underscore-dangle */ /* eslint-disable no-console */ -/* eslint-disable @typescript-eslint/no-unused-expressions */ import { DataSource } from 'typeorm' import { HttpService } from '@nestjs/axios' import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common' @@ -16,7 +14,6 @@ import { TokenRankListData, } from './marketcap.dto' import Wiki from '../../Database/Entities/wiki.entity' -import Tag from '../../Database/Entities/tag.entity' import WikiService from '../Wiki/wiki.service' import MarketCapIds from '../../Database/Entities/marketCapIds.entity' @@ -39,7 +36,7 @@ interface RankPageWiki { @Injectable() class MarketCapService { - private RANK_LIMIT = 500 + private RANK_LIMIT = 1000 private API_KEY: string @@ -67,96 +64,146 @@ class MarketCapService { ): Promise { const wikiRepository = this.dataSource.getRepository(Wiki) const marketCapIdRepository = this.dataSource.getRepository(MarketCapIds) - const marketCapId = await marketCapIdRepository.findOne({ - where: { coingeckoId: id, kind: category as RankType }, - }) - const noCategoryId = marketCapId?.wikiId || id + const baseCoingeckoUrl = 'https://www.coingecko.com/en' + const coingeckoProfileUrl = `${baseCoingeckoUrl}/${ + category === 'cryptocurrencies' ? 'coins' : 'nft' + }/${id}` + + const cachedWiki: RankPageWiki | undefined = await this.cacheManager.get(id) + if (!cachedWiki) { + const marketCapId = await marketCapIdRepository.findOne({ + where: { coingeckoId: id, kind: category as RankType }, + select: ['wikiId'], + }) - const wiki = - (await this.findWikiByCoingeckoUrl(id, category, marketCapId?.wikiId)) || - (await wikiRepository - .createQueryBuilder('wiki') - .leftJoinAndSelect('wiki.wikiEvents', 'events') - .where('wiki.id = :id AND wiki.hidden = false', { - id: noCategoryId, - }) - .getOne()) || - (await wikiRepository + const noCategoryId = marketCapId?.wikiId || id + + const baseQuery = wikiRepository .createQueryBuilder('wiki') - .leftJoinAndSelect('wiki.wikiEvents', 'events') - .innerJoinAndSelect( - 'wiki.categories', - 'category', - 'category.id = :categoryId', - { - categoryId: category, - }, - ) - .where('wiki.id = :id AND wiki.hidden = false', { id }) - .getOne()) + .select('wiki.id') + .addSelect('wiki.title') + .addSelect('wiki.ipfs') + .addSelect('wiki.images') + + const wikiQuery = baseQuery + .clone() + .addSelect('wiki.metadata') + .addSelect('wiki.created') + .addSelect('wiki.linkedWikis') + .addSelect('events.type') + .addSelect('events.date') + .leftJoin('wiki.wikiEvents', 'events') + .leftJoinAndSelect('wiki.tags', 'tags') + + const wiki = + (await wikiQuery + .andWhere( + `EXISTS ( + SELECT 1 + FROM json_array_elements(wiki.metadata) AS meta + WHERE meta->>'id' = 'coingecko_profile' AND meta->>'value' = :url + )`, + { url: coingeckoProfileUrl }, + ) + .where('wiki.id = :id AND wiki.hidden = false', { id }) + .getOne()) || + (await wikiQuery + .where('wiki.id = :id AND wiki.hidden = false', { + id: noCategoryId, + }) + .getOne()) || + (await wikiQuery + .innerJoin( + 'wiki.categories', + 'category', + 'category.id = :categoryId', + { + categoryId: category, + }, + ) + .where('wiki.id = :id AND wiki.hidden = false', { id }) + .getOne()) + + const [founders, blockchain] = await Promise.all([ + (async () => { + if (wiki?.linkedWikis?.founders) { + const founderResults = [] + for (const f of wiki.linkedWikis.founders) { + const result = await baseQuery + .where('wiki.id = :id AND wiki.hidden = false', { + id: f, + }) + .getOne() + if (result) { + founderResults.push(result) + } + } + return founderResults + } + return [] + })(), + (async () => { + if (wiki?.linkedWikis?.blockchains) { + const blockchainResults = [] + for (const b of wiki.linkedWikis.blockchains) { + const result = await baseQuery + .where('wiki.id = :id AND wiki.hidden = false', { + id: b, + }) + .getOne() + if (result) { + blockchainResults.push(result) + } + } + return blockchainResults + } + return [] + })(), + ]) + + const result = { wiki, founders, blockchain } + + await this.cacheManager.set(id, result, { + ttl: 3600, + }) - const tag = await this.getTags(noCategoryId) - const wikiAndTags = { - ...wiki, - tags: [...tag], + return result as unknown as RankPageWiki } - const wikiResult = wiki && tag ? wikiAndTags : wiki - const [founders, blockchain] = await Promise.all([ - this.wikiService.getFullLinkedWikis( - wikiResult?.linkedWikis?.founders as string[], - ) || [], - this.wikiService.getFullLinkedWikis( - wikiResult?.linkedWikis?.blockchains as string[], - ) || [], - ]) - - const result = { wiki: wikiResult, founders, blockchain } - return result as RankPageWiki - } - private async getTags(id: string) { - const ds = this.dataSource.getRepository(Tag) - return ds.query( - ` - SELECT - "tags"."id" - FROM "tag" "tags" - INNER JOIN "wiki_tags_tag" "wiki_tags_tag" - ON "wiki_tags_tag"."wikiId" IN ($1) - AND "wiki_tags_tag"."tagId"="tags"."id" - `, - [id], - ) + return cachedWiki } async getWikiData( coinsData: Record | undefined, kind: RankType, - category?: string, ): Promise { - kind.toLowerCase() - const wikiPromises = coinsData?.map((element: any) => - this.findWiki(element.id, kind), - ) + const k = kind.toLowerCase() - let wikis: RankPageWiki[] | undefined = await this.cacheManager.get( - `wiki-${kind}-${category || ''}`, - ) + const batchSize = 50 + const allWikis: (RankPageWiki | null)[] = [] - if (!wikis) { - wikis = await Promise.all(wikiPromises) - this.cacheManager.set(`wiki-${kind}`, wikis, { - ttl: 3600, - }) + if (coinsData) { + for (let i = 0; i < coinsData.length; i += batchSize) { + const batch = coinsData.slice(i, i + batchSize) + + const batchPromises = batch.map((element: any) => + this.findWiki(element.id, k), + ) + const batchWikis = await Promise.all(batchPromises) + + allWikis.push(...batchWikis) + } } - return wikis + + return allWikis as RankPageWiki[] } - async marketData(kind: RankType, category?: TokenCategory) { - const categoryParam = category ? `category=${category}&` : '' - const data = await this.cgMarketDataApiCall(kind, categoryParam) - const wikis = await this.getWikiData(data, kind, category) + async marketData(args: MarketCapInputs) { + const { kind } = args + + const data = await this.cgMarketDataApiCall(args) + const wikis = await this.getWikiData(data, kind) const processElement = (element: any, rankpageWiki: any) => { const tokenData = @@ -208,6 +255,7 @@ class MarketCapService { return { ...rankpageWiki.wiki, events: rankpageWiki.wiki.__wikiEvents__, + tags: rankpageWiki.wiki.__tags__, founderWikis: rankpageWiki.founders, blockchainWikis: rankpageWiki.blockchain, ...marketData, @@ -224,60 +272,67 @@ class MarketCapService { } async cgMarketDataApiCall( - kind: RankType, - categoryParam?: string, + args: MarketCapInputs, ): Promise | undefined> { + const { kind, category, limit, offset } = args + const categoryParam = category ? `category=${category}&` : '' + const baseUrl = 'https://pro-api.coingecko.com/api/v3/' - const perPage = 50 + + const perPage = limit || 250 const totalPages = Math.ceil(this.RANK_LIMIT / perPage) + const pageToFetch = offset ? Math.ceil(offset / perPage) + 1 : 1 + const allData = [] try { - for (let page = 1; page <= totalPages; page += 1) { + for (let page = pageToFetch; page <= totalPages; page += 1) { const url = kind === RankType.TOKEN ? `${baseUrl}coins/markets?vs_currency=usd&${categoryParam}order=market_cap_desc&per_page=${perPage}&page=${page}` : `${baseUrl}nfts/markets?order=h24_volume_usd_desc&per_page=${perPage}&page=${page}` - const response = await this.httpService - .get(url, { - headers: { - 'x-cg-pro-api-key': this.API_KEY, - }, - }) - .toPromise() - - if (response?.data) { - allData.push(...response.data) + const finalCachedResult: any | undefined = await this.cacheManager.get( + url, + ) + if (finalCachedResult) { + allData.push(...finalCachedResult) + break + } else { + const response = await this.httpService + .get(url, { + headers: { + 'x-cg-pro-api-key': this.API_KEY, + }, + }) + .toPromise() + if (response?.data) { + allData.push(...response.data) + await this.cacheManager.set(url, allData, { ttl: 180 }) + } + } + if (allData.length >= limit) { + break } } } catch (err: any) { console.error(err.message) } + return allData.slice(0, this.RANK_LIMIT) } async ranks( args: MarketCapInputs, - search = false, - ): Promise { - const key = this.getCacheKey(args) - let result - const finalCachedResult: any | undefined = await this.cacheManager.get(key) - if (finalCachedResult) { - result = finalCachedResult - } else { - const data = await this.marketData(args.kind, args.category) - - result = - args.kind === RankType.NFT - ? (data as unknown as NftRankListData) - : (data as unknown as TokenRankListData) - - await this.cacheManager.set(key, result, { ttl: 180 }) - } + ): Promise<(TokenRankListData | NftRankListData)[]> { + const data = await this.marketData(args) - return search ? result : result.slice(args.offset, args.offset + args.limit) + const result = + args.kind === RankType.NFT + ? (data as unknown as NftRankListData[]) + : (data as unknown as TokenRankListData[]) + + return result } getCacheKey(args: MarketCapInputs) { @@ -334,7 +389,7 @@ class MarketCapService { async wildcardSearch(args: MarketCapInputs) { if (!args.search) return [] - const cache = (await this.ranks(args, true)) as unknown as ( + const cache = (await this.ranks(args)) as unknown as ( | TokenRankListData | NftRankListData )[] @@ -352,44 +407,6 @@ class MarketCapService { return (nftMatch as NftRankListData) || (tokenMatch as TokenRankListData) }) } - - async findWikiByCoingeckoUrl( - coingeckoId: string, - category: string, - id?: string, - ): Promise { - const wikiRepository = this.dataSource.getRepository(Wiki) - const baseCoingeckoUrl = 'https://www.coingecko.com/en' - const coingeckoProfileUrl = `${baseCoingeckoUrl}/${ - category === 'cryptocurrencies' ? 'coins' : 'nft' - }/${coingeckoId}` - - const queryBuilder = wikiRepository - .createQueryBuilder('wiki') - .where('wiki.hidden = false') - .andWhere( - `EXISTS ( - SELECT 1 - FROM json_array_elements(wiki.metadata) AS meta - WHERE meta->>'id' = 'coingecko_profile' AND meta->>'value' = :url - )`, - { url: coingeckoProfileUrl }, - ) - .innerJoinAndSelect('wiki.wikiEvents', 'events') - .innerJoinAndSelect( - 'wiki.categories', - 'category', - 'category.id = :categoryId', - { categoryId: category }, - ) - - if (id !== undefined) { - queryBuilder.andWhere('wiki.id = :wikiId', { wikiId: id }) - } - - const wiki = await queryBuilder.getOne() - return wiki || null - } } export default MarketCapService diff --git a/src/App/revalidatePage/revalidatePage.service.ts b/src/App/revalidatePage/revalidatePage.service.ts index 51dc8fa3..f41403c9 100644 --- a/src/App/revalidatePage/revalidatePage.service.ts +++ b/src/App/revalidatePage/revalidatePage.service.ts @@ -3,6 +3,8 @@ import { Injectable, CACHE_MANAGER, Inject } from '@nestjs/common' import { ConfigService } from '@nestjs/config' import { DataSource } from 'typeorm' import { Cache } from 'cache-manager' +import { Cron, CronExpression } from '@nestjs/schedule' +import { firstValueFrom } from 'rxjs' import { RankType } from '../marketCap/marketcap.dto' import Category from '../../Database/Entities/category.entity' import Wiki from '../../Database/Entities/wiki.entity' @@ -29,6 +31,8 @@ export interface RevalidateStatus { @Injectable() export class RevalidatePageService { + private failedUrls = new Map() + constructor( private httpService: HttpService, private dataSource: DataSource, @@ -56,22 +60,56 @@ export class RevalidatePageService { path += `/${id}` } - try { - await Promise.all([ - this.httpService.get(`${revalidateUrl}&path=/en${path}`).toPromise(), - this.httpService.get(`${revalidateUrl}&path=/zh${path}`).toPromise(), - this.httpService.get(`${revalidateUrl}&path=/ko${path}`).toPromise(), - ]) - } catch (e: any) { - console.error( - 'Error revalidating path', - e.response?.config?.url || e.message, - ) - } - + const urlsToRevalidate = [ + `${revalidateUrl}&path=/en${path}`, + `${revalidateUrl}&path=/zh${path}`, + `${revalidateUrl}&path=/ko${path}`, + ] + + await Promise.all( + urlsToRevalidate.map(async (urlToRevalidate) => { + try { + await this.httpService.get(urlToRevalidate).toPromise() + } catch (e: any) { + console.error( + 'Error revalidating path', + e.response?.config?.url || e.message, + urlToRevalidate, + ) + this.failedUrls.set(urlToRevalidate, 3) + } + }), + ) return true } + @Cron(CronExpression.EVERY_10_SECONDS) + async retryFailedUrl() { + for (const [k, v] of this.failedUrls) { + if (v > 0) { + try { + await Promise.all( + k.map(async (url: string) => { + const { data } = await firstValueFrom( + this.httpService.get(url).pipe(), + ) + + if (data && data.revalidated === true) { + console.log(`♻️ ♻️ Successfully revalidated: ${url}`) + } + }), + ) + + this.failedUrls.delete(k) + } catch (error: any) { + this.failedUrls.set(k, v - 1) + } + } else { + this.failedUrls.delete(k) + } + } + } + async revalidatePage( page: RevalidateEndpoints, id?: string, diff --git a/src/App/utils/discordWebhookHandler.ts b/src/App/utils/discordWebhookHandler.ts index 2c856785..e0aa4999 100644 --- a/src/App/utils/discordWebhookHandler.ts +++ b/src/App/utils/discordWebhookHandler.ts @@ -133,7 +133,7 @@ export default class WebhookHandler { { color: 0xff9900, title: `📢 Wiki report on ${payload?.urlId} 📢`, - url: `${this.getWebpageUrl()}wiki/${payload?.urlId}`, + url: `${this.getWebpageUrl()}/wiki/${payload?.urlId}`, description: `${payload?.description}`, footer: { text: `Flagged by ${user?.username || 'user'}`, diff --git a/src/App/utils/test-helpers/testHelpers.ts b/src/App/utils/test-helpers/testHelpers.ts index 4e8df33c..1f6350ee 100644 --- a/src/App/utils/test-helpers/testHelpers.ts +++ b/src/App/utils/test-helpers/testHelpers.ts @@ -105,7 +105,7 @@ export enum ProviderEnum { wikiResolver = 'wikiResolver', eventEmitter2 = 'eventEmitter2', configService = 'configService', - posHogService = 'postHogService', + postHogService = 'postHogService', tokenValidator = 'tokenValidator', tagRepository = 'tagRepository', dbStoreService = 'dbStoreService', diff --git a/src/Database/Entities/explorer.entity.ts b/src/Database/Entities/explorer.entity.ts new file mode 100644 index 00000000..64e17336 --- /dev/null +++ b/src/Database/Entities/explorer.entity.ts @@ -0,0 +1,27 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm' +import { ArgsType, Field, ID, ObjectType } from '@nestjs/graphql' + +@ObjectType() +@ArgsType() +@Entity() +class Explorer { + @Field(() => ID) + @PrimaryColumn('varchar', { + length: 255, + }) + id!: string + + @Field(() => String) + @Column('varchar') + explorer!: string + + @Field(() => String) + @Column('varchar') + baseUrl!: string + + @Field(() => Boolean) + @Column('boolean', { default: false }) + hidden!: boolean +} + +export default Explorer diff --git a/src/Database/database.module.ts b/src/Database/database.module.ts index 030f8a9f..f7af5724 100644 --- a/src/Database/database.module.ts +++ b/src/Database/database.module.ts @@ -21,6 +21,7 @@ import IQHolderAddress from './Entities/iqHolderAddress.entity' import IQHolder from './Entities/iqHolder.entity' import MarketCapIds from './Entities/marketCapIds.entity' import Events from './Entities/Event.entity' +import Explorer from './Entities/explorer.entity' @Module({ imports: [ @@ -54,6 +55,7 @@ import Events from './Entities/Event.entity' IQHolderAddress, MarketCapIds, Events, + Explorer ], synchronize: true, keepConnectionAlive: true, diff --git a/src/Indexer/IndexerWebhook/controllers/indexerWebhook.controller.spec.ts b/src/Indexer/IndexerWebhook/controllers/indexerWebhook.controller.spec.ts index 58dc16ea..0cd8ac91 100644 --- a/src/Indexer/IndexerWebhook/controllers/indexerWebhook.controller.spec.ts +++ b/src/Indexer/IndexerWebhook/controllers/indexerWebhook.controller.spec.ts @@ -53,7 +53,7 @@ describe('IndexerWebhookController', () => { providers: [ ...getProviders([ ProviderEnum.runCommand, - ProviderEnum.posHogService, + ProviderEnum.postHogService, ProviderEnum.graphProviderService, ProviderEnum.ipfsGetterService, ProviderEnum.ipfsValidatorService, diff --git a/src/Relayer/services/relayer.service.ts b/src/Relayer/services/relayer.service.ts index 60263910..da2065bf 100644 --- a/src/Relayer/services/relayer.service.ts +++ b/src/Relayer/services/relayer.service.ts @@ -8,11 +8,11 @@ import { ConfigService } from '@nestjs/config' import { catchError, firstValueFrom } from 'rxjs' import { HttpService } from '@nestjs/axios' import { AxiosError } from 'axios' +import { PosthogService } from 'nestjs-posthog' import WikiAbi from '../utils/wiki.abi' import USER_ACTIVITY_LIMIT from '../../globalVars' import ActivityRepository from '../../App/Activities/activity.repository' import AppService from '../../App/app.service' -import { PosthogService } from 'nestjs-posthog' @Injectable() class RelayerService { diff --git a/src/posthog/posthog.module.ts b/src/posthog/posthog.module.ts index 7096b4ee..82525757 100644 --- a/src/posthog/posthog.module.ts +++ b/src/posthog/posthog.module.ts @@ -17,4 +17,4 @@ import { ConfigModule, ConfigService } from '@nestjs/config' }), ], }) -export default class PosHogManager {} +export default class PostHogManager {} From 4aaacce4d5669ffd03d14b7d7f67e2d780f10c12 Mon Sep 17 00:00:00 2001 From: icemedia001 Date: Tue, 24 Sep 2024 00:20:11 +0100 Subject: [PATCH 6/7] Updated relayer module for posthog implementation --- src/Relayer/relayer.module.ts | 18 +++++++++++++++--- src/Relayer/services/relayer.service.ts | 1 + 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/Relayer/relayer.module.ts b/src/Relayer/relayer.module.ts index 3352b017..5a795d7f 100644 --- a/src/Relayer/relayer.module.ts +++ b/src/Relayer/relayer.module.ts @@ -1,16 +1,28 @@ import { Module } from '@nestjs/common' +import { PosthogModule, PosthogService } from 'nestjs-posthog' +import { ConfigModule, ConfigService } from '@nestjs/config' import RelayerService from './services/relayer.service' import RelayerController from './controllers/relayer.controller' import RelayerResolver from './resolvers/relayer.resolver' import httpModule from '../httpModule' import ActivityModule from '../App/Activities/activity.module' import AppService from '../App/app.service' -import { POSTHOG_MODULE_OPTIONS, PosthogModule } from 'nestjs-posthog' @Module({ - imports: [httpModule(10000), ActivityModule, PosthogModule], + imports: [httpModule(10000), ActivityModule, + PosthogModule.forRootAsync({ + imports: [ConfigModule], + inject: [ConfigService], + useFactory: (config: ConfigService) => ({ + apiKey: config.get('POSTHOG_API_KEY') as string, + options: { + host: config.get('POSTHOG_API_URL') as string, + }, + mock: false, + }), + }),], controllers: [RelayerController], - providers: [RelayerService, RelayerResolver, AppService], + providers: [RelayerService, RelayerResolver, AppService, PosthogService], }) class RelayerModule {} diff --git a/src/Relayer/services/relayer.service.ts b/src/Relayer/services/relayer.service.ts index da2065bf..dbfd134f 100644 --- a/src/Relayer/services/relayer.service.ts +++ b/src/Relayer/services/relayer.service.ts @@ -181,6 +181,7 @@ class RelayerService { } return result } + } export default RelayerService From fc9b3d760825b7bff7595fa4209d9c956494e583 Mon Sep 17 00:00:00 2001 From: icemedia001 Date: Tue, 24 Sep 2024 00:22:20 +0100 Subject: [PATCH 7/7] Updated relayer module for posthog implementation --- src/Relayer/services/relayer.service.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Relayer/services/relayer.service.ts b/src/Relayer/services/relayer.service.ts index dbfd134f..da2065bf 100644 --- a/src/Relayer/services/relayer.service.ts +++ b/src/Relayer/services/relayer.service.ts @@ -181,7 +181,6 @@ class RelayerService { } return result } - } export default RelayerService