From 7546215341e1fd06e960f07504ed4f96c7b39581 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 14 Jul 2025 12:07:28 +0200 Subject: [PATCH 001/169] emiiter engine --- .../src/emitters/AbstractEmitterEngine.ts | 0 .../src/emitters/EmitterEngine.ts | 96 +++++++++++++++++++ .../src/emitters/emitter-interfaces.ts | 47 +++++++++ .../service-core/src/emitters/event-error.ts | 10 ++ .../src/emitters/events/connect-event.ts | 34 +++++++ .../src/emitters/events/disconnect-event.ts | 36 +++++++ .../service-core/src/emitters/events/index.ts | 4 + .../src/routes/endpoints/sync-stream.ts | 27 +++++- .../service-core/src/system/ServiceContext.ts | 12 ++- 9 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 packages/service-core/src/emitters/AbstractEmitterEngine.ts create mode 100644 packages/service-core/src/emitters/EmitterEngine.ts create mode 100644 packages/service-core/src/emitters/emitter-interfaces.ts create mode 100644 packages/service-core/src/emitters/event-error.ts create mode 100644 packages/service-core/src/emitters/events/connect-event.ts create mode 100644 packages/service-core/src/emitters/events/disconnect-event.ts create mode 100644 packages/service-core/src/emitters/events/index.ts diff --git a/packages/service-core/src/emitters/AbstractEmitterEngine.ts b/packages/service-core/src/emitters/AbstractEmitterEngine.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts new file mode 100644 index 000000000..1904797be --- /dev/null +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -0,0 +1,96 @@ +import { BaseEmitterEngine, EmitterEvent, EmitterEventData, EventNames, isEventError } from './emitter-interfaces.js'; +import * as storage from '../storage/storage-index.js'; +import EventEmitter from 'node:events'; +import { logger } from '@powersync/lib-services-framework'; +import { EventError } from './event-error.js'; + +export abstract class AbstractEmitterEngine { + private emitter: EventEmitter; + storageEngine: storage.StorageEngine; + eventsMap: Map = new Map(); + + protected constructor(storage: storage.StorageEngine) { + this.emitter = new EventEmitter({ captureRejections: true }); + this.storageEngine = storage; + this.emitter.on('error', (error: Error | EventError) => { + if (isEventError(error) && this.eventsMap.has(error.eventName)) { + const event = this.eventsMap.get(error.eventName)!; + const errorHandler = + event.errorhandler ?? + ((error) => { + logger.error(error.message); + }); + errorHandler(error); + } + }); + } + + get events(): EventNames[] { + return this.emitter.eventNames() as EventNames[]; + } + + protected bindEvents(events: EmitterEvent[]): void { + const eventNames = this.emitter.eventNames(); + for (const event of events) { + if (!eventNames.includes(event.name)) { + this.eventsMap.set(event.name, event); + this.emitter.on(event.name, event.handler(this.storageEngine).bind(event)); + } else { + logger.warn(`Event ${event.name} is already registered. Skipping.`); + } + } + } + + protected emit(eventName: EventNames, data: EmitterEventData): void { + if (!this.emitter.eventNames().includes(eventName)) { + throw new Error(`Event ${eventName} is not registered.`); + } + this.emitter.emit(eventName, data); + } + + protected removeListeners(eventName?: EventNames): void { + if (eventName) { + this.emitter.removeAllListeners(eventName); + logger.info(`Removed all listeners for event: ${eventName}`); + } else { + this.stop(); + } + } + + protected stop(): void { + this.emitter.removeAllListeners(); + logger.info('Emitter engine shut down and all listeners removed.'); + } +} + +export class EmitterEngine extends AbstractEmitterEngine implements BaseEmitterEngine { + constructor(events: EmitterEvent[], storageRef: storage.StorageEngine) { + super(storageRef); + this.bindEvents(events); + } + + event(eventName: EventNames): EmitterEvent { + const event = this.eventsMap.get(eventName); + if (!event) { + throw new Error(`Event ${eventName} is not registered.`); + } + return event; + } + + removeListeners(eventName?: EventNames): void { + if (eventName) { + this.removeListeners(eventName); + } + } + eventNames(): EventNames[] { + return this.events; + } + + emitEvent(eventName: EventNames, data: EmitterEventData): void { + return this.emit(eventName, data); + } + + async shutDown(): Promise { + this.stop(); + } +} diff --git a/packages/service-core/src/emitters/emitter-interfaces.ts b/packages/service-core/src/emitters/emitter-interfaces.ts new file mode 100644 index 000000000..7ab57edf7 --- /dev/null +++ b/packages/service-core/src/emitters/emitter-interfaces.ts @@ -0,0 +1,47 @@ +import { JwtPayload } from '../auth/JwtPayload.js'; +import * as storage from '../storage/storage-index.js'; +import { EventError } from './event-error.js'; + +export enum EventNames { + // SdK events + SDK_CONNECT_EVENT = 'sdk-connect-event', + SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' +} + +export type SdkEvent = { + client_id?: string; + user_id: string; + user_agent: string; + jwt_token?: JwtPayload; +}; + +export type EventConnectData = { + type: EventNames.SDK_CONNECT_EVENT; + connect_at: number; +} & SdkEvent; + +export type EventDisconnectData = { + type: EventNames.SDK_DISCONNECT_EVENT; + disconnect_at: number; +} & SdkEvent; + +export type EmitterEventData = EventConnectData | EventDisconnectData; +export type EventHandler = (data: EmitterEventData) => Promise; +export type StorageHandler = (storageEngine: storage.StorageEngine) => EventHandler; +export interface EmitterEvent { + name: EventNames; + handler: StorageHandler; + errorhandler?: (error: Error | EventError) => void; +} + +export interface BaseEmitterEngine { + eventNames(): EventNames[]; + emitEvent(eventName: EventNames, data: EmitterEventData): void; + removeListeners(eventName?: EventNames): void; + event(eventName: EventNames): EmitterEvent; + shutDown(): Promise; +} + +export function isEventError(error: Error | EventError): error is EventError { + return (error as EventError).eventName !== undefined; +} diff --git a/packages/service-core/src/emitters/event-error.ts b/packages/service-core/src/emitters/event-error.ts new file mode 100644 index 000000000..438f13cf8 --- /dev/null +++ b/packages/service-core/src/emitters/event-error.ts @@ -0,0 +1,10 @@ +import { EventNames } from './emitter-interfaces.js'; + +export class EventError extends Error { + constructor( + public eventName: EventNames, + message: string + ) { + super(message); + } +} diff --git a/packages/service-core/src/emitters/events/connect-event.ts b/packages/service-core/src/emitters/events/connect-event.ts new file mode 100644 index 000000000..c99c9fbbd --- /dev/null +++ b/packages/service-core/src/emitters/events/connect-event.ts @@ -0,0 +1,34 @@ +import { EmitterEvent, EmitterEventData, EventConnectData, EventHandler, EventNames } from '../emitter-interfaces.js'; +import * as storage from '../../storage/storage-index.js'; +import { EventError } from '../event-error.js'; +import { logger } from '@powersync/lib-services-framework'; + +export class ConnectEvent implements EmitterEvent { + private type = EventNames.SDK_CONNECT_EVENT; + get name(): EventNames { + return this.type; + } + errorhandler(error: Error | EventError): void { + if (error instanceof EventError) { + logger.error(`EventError in ${error.eventName}:`, error.message); + } else { + logger.error(error.message); + } + } + handler(storageEngine: storage.StorageEngine): EventHandler { + // TODO: USE STORAGE ENGINE + const storage = storageEngine.activeStorage.storage; + return async (data: EmitterEventData) => { + try { + const disconnectData = data as EventConnectData; + console.log( + `Connect event triggered for user: ${disconnectData.user_id} at ${new Date(disconnectData.connect_at).toISOString()}` + ); + } catch (error) { + throw new EventError(this.type, error.message); + } + }; + } +} + +export const connectEvent = new ConnectEvent(); diff --git a/packages/service-core/src/emitters/events/disconnect-event.ts b/packages/service-core/src/emitters/events/disconnect-event.ts new file mode 100644 index 000000000..2060f6bfd --- /dev/null +++ b/packages/service-core/src/emitters/events/disconnect-event.ts @@ -0,0 +1,36 @@ +import { + EmitterEvent, + EmitterEventData, + EventDisconnectData, + EventHandler, + EventNames +} from '../emitter-interfaces.js'; +import * as storage from '../../storage/storage-index.js'; +import { EventError } from '../event-error.js'; +import { logger } from '@powersync/lib-services-framework'; + +export class DisconnectEvent implements EmitterEvent { + private type = EventNames.SDK_DISCONNECT_EVENT; + get name(): EventNames { + return this.type; + } + errorhandler(error: Error | EventError): void { + if (error instanceof EventError) { + logger.error(`EventError in ${error.eventName}:`, error.message); + } else { + logger.error(error.message); + } + } + handler(storageEngine: storage.StorageEngine): EventHandler { + // TODO: USE STORAGE ENGINE + const storage = storageEngine.activeStorage.storage; + return async (data: EmitterEventData) => { + const disconnectData = data as EventDisconnectData; + console.log( + `Disconnect event triggered for user: ${disconnectData.user_id} at ${new Date(disconnectData.disconnect_at).toISOString()}` + ); + }; + } +} + +export const disconnectEvent = new DisconnectEvent(); diff --git a/packages/service-core/src/emitters/events/index.ts b/packages/service-core/src/emitters/events/index.ts new file mode 100644 index 000000000..a2b57baff --- /dev/null +++ b/packages/service-core/src/emitters/events/index.ts @@ -0,0 +1,4 @@ +import { disconnectEvent } from './disconnect-event.js'; +import { connectEvent } from './connect-event.js'; + +export const events = [disconnectEvent, connectEvent]; diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 644627ede..22170da87 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -1,4 +1,4 @@ -import { ErrorCode, errors, logger, router, schema } from '@powersync/lib-services-framework'; +import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework'; import { RequestParameters } from '@powersync/service-sync-rules'; import { Readable } from 'stream'; @@ -9,6 +9,7 @@ import { authUser } from '../auth.js'; import { routeDefinition } from '../router.js'; import { APIMetric } from '@powersync/service-types'; +import { EventNames } from '../../emitters/emitter-interfaces.js'; export enum SyncRoutes { STREAM = '/sync/stream' @@ -20,7 +21,7 @@ export const syncStreamed = routeDefinition({ authorize: authUser, validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }), handler: async (payload) => { - const { service_context, logger } = payload.context; + const { service_context, logger, token_payload } = payload.context; const { routerEngine, storageEngine, metricsEngine, syncContext } = service_context; const headers = payload.request.headers; const userAgent = headers['x-user-agent'] ?? headers['user-agent']; @@ -34,6 +35,13 @@ export const syncStreamed = routeDefinition({ user_id: payload.context.user_id }; + const sdkData = { + client_id: clientId, + user_id: payload.context.user_id!, + user_agent: userAgent as string, + jwt_token: token_payload + }; + if (routerEngine.closed) { throw new errors.ServiceError({ status: 503, @@ -61,6 +69,11 @@ export const syncStreamed = routeDefinition({ const tracker = new sync.RequestTracker(metricsEngine); try { metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); + service_context.emitterEngine.emitEvent(EventNames.SDK_CONNECT_EVENT, { + type: EventNames.SDK_CONNECT_EVENT, + ...sdkData, + connect_at: streamStart + }); const stream = Readable.from( sync.transformToBytesTracked( sync.ndjson( @@ -123,6 +136,11 @@ export const syncStreamed = routeDefinition({ } controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); + service_context.emitterEngine.emitEvent(EventNames.SDK_DISCONNECT_EVENT, { + type: EventNames.SDK_DISCONNECT_EVENT, + ...sdkData, + disconnect_at: Date.now() + }); logger.info(`Sync stream complete`, { ...tracker.getLogMeta(), stream_ms: Date.now() - streamStart, @@ -133,6 +151,11 @@ export const syncStreamed = routeDefinition({ } catch (ex) { controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); + service_context.emitterEngine.emitEvent(EventNames.SDK_DISCONNECT_EVENT, { + type: EventNames.SDK_DISCONNECT_EVENT, + ...sdkData, + disconnect_at: Date.now() + }); } } }); diff --git a/packages/service-core/src/system/ServiceContext.ts b/packages/service-core/src/system/ServiceContext.ts index fa1d68f7d..830daadef 100644 --- a/packages/service-core/src/system/ServiceContext.ts +++ b/packages/service-core/src/system/ServiceContext.ts @@ -1,4 +1,4 @@ -import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework'; +import { container, LifeCycledSystem, MigrationManager, ServiceIdentifier } from '@powersync/lib-services-framework'; import { framework } from '../index.js'; import * as metrics from '../metrics/MetricsEngine.js'; @@ -8,6 +8,8 @@ import * as routes from '../routes/routes-index.js'; import * as storage from '../storage/storage-index.js'; import { SyncContext } from '../sync/SyncContext.js'; import * as utils from '../util/util-index.js'; +import { EmitterEngine } from '../emitters/EmitterEngine.js'; +import { events } from '../emitters/events/index.js'; export interface ServiceContext { configuration: utils.ResolvedPowerSyncConfig; @@ -19,6 +21,7 @@ export interface ServiceContext { migrations: PowerSyncMigrationManager; syncContext: SyncContext; serviceMode: ServiceContextMode; + emitterEngine: EmitterEngine; } export enum ServiceContextMode { @@ -45,6 +48,7 @@ export class ServiceContextContainer implements ServiceContext { configuration: utils.ResolvedPowerSyncConfig; lifeCycleEngine: LifeCycledSystem; storageEngine: storage.StorageEngine; + emitterEngine: EmitterEngine; syncContext: SyncContext; routerEngine: routes.RouterEngine; serviceMode: ServiceContextMode; @@ -66,6 +70,8 @@ export class ServiceContextContainer implements ServiceContext { } }); + this.emitterEngine = new EmitterEngine(events, this.storageEngine); + this.lifeCycleEngine.withLifecycle(this.storageEngine, { start: (storageEngine) => storageEngine.start(), stop: (storageEngine) => storageEngine.shutDown() @@ -89,6 +95,10 @@ export class ServiceContextContainer implements ServiceContext { // Migrations should be executed before the system starts start: () => migrationManager[Symbol.asyncDispose]() }); + + this.lifeCycleEngine.withLifecycle(this.emitterEngine, { + stop: (emitterEngine) => emitterEngine.shutDown() + }); } get replicationEngine(): replication.ReplicationEngine | null { From edee67ab4aaf072020a84092dc25ccaed2479a2c Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 14 Jul 2025 13:42:23 +0200 Subject: [PATCH 002/169] emitter changes --- .../src/emitters/AbstractEmitterEngine.ts | 64 +++++++++++++++++ .../src/emitters/EmitterEngine.ts | 71 +++---------------- 2 files changed, 72 insertions(+), 63 deletions(-) diff --git a/packages/service-core/src/emitters/AbstractEmitterEngine.ts b/packages/service-core/src/emitters/AbstractEmitterEngine.ts index e69de29bb..e1e20b301 100644 --- a/packages/service-core/src/emitters/AbstractEmitterEngine.ts +++ b/packages/service-core/src/emitters/AbstractEmitterEngine.ts @@ -0,0 +1,64 @@ +import { EmitterEvent, EmitterEventData, EventNames, isEventError } from './emitter-interfaces.js'; +import * as storage from '../storage/storage-index.js'; +import EventEmitter from 'node:events'; +import { logger } from '@powersync/lib-services-framework'; +import { EventError } from './event-error.js'; + +export abstract class AbstractEmitterEngine { + private emitter: EventEmitter; + storageEngine: storage.StorageEngine; + eventsMap: Map = new Map(); + + protected constructor(storage: storage.StorageEngine) { + this.emitter = new EventEmitter({ captureRejections: true }); + this.storageEngine = storage; + this.emitter.on('error', (error: Error | EventError) => { + if (isEventError(error) && this.eventsMap.has(error.eventName)) { + const event = this.eventsMap.get(error.eventName)!; + const errorHandler = + event.errorhandler ?? + ((error) => { + logger.error(error.message); + }); + errorHandler(error); + } + }); + } + + get events(): EventNames[] { + return this.emitter.eventNames() as EventNames[]; + } + + protected bindEvents(events: EmitterEvent[]): void { + const eventNames = this.emitter.eventNames(); + for (const event of events) { + if (!eventNames.includes(event.name)) { + this.eventsMap.set(event.name, event); + this.emitter.on(event.name, event.handler(this.storageEngine).bind(event)); + } else { + logger.warn(`Event ${event.name} is already registered. Skipping.`); + } + } + } + + protected emit(eventName: EventNames, data: EmitterEventData): void { + if (!this.emitter.eventNames().includes(eventName)) { + throw new Error(`Event ${eventName} is not registered.`); + } + this.emitter.emit(eventName, data); + } + + protected removeListeners(eventName?: EventNames): void { + if (eventName) { + this.emitter.removeAllListeners(eventName); + logger.info(`Removed all listeners for event: ${eventName}`); + } else { + this.stop(); + } + } + + protected stop(): void { + this.emitter.removeAllListeners(); + logger.info('Emitter engine shut down and all listeners removed.'); + } +} diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 1904797be..32632925e 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -1,71 +1,14 @@ -import { BaseEmitterEngine, EmitterEvent, EmitterEventData, EventNames, isEventError } from './emitter-interfaces.js'; +import { BaseEmitterEngine, EmitterEvent, EmitterEventData, EventNames } from './emitter-interfaces.js'; import * as storage from '../storage/storage-index.js'; -import EventEmitter from 'node:events'; +import { AbstractEmitterEngine } from './AbstractEmitterEngine.js'; import { logger } from '@powersync/lib-services-framework'; -import { EventError } from './event-error.js'; - -export abstract class AbstractEmitterEngine { - private emitter: EventEmitter; - storageEngine: storage.StorageEngine; - eventsMap: Map = new Map(); - - protected constructor(storage: storage.StorageEngine) { - this.emitter = new EventEmitter({ captureRejections: true }); - this.storageEngine = storage; - this.emitter.on('error', (error: Error | EventError) => { - if (isEventError(error) && this.eventsMap.has(error.eventName)) { - const event = this.eventsMap.get(error.eventName)!; - const errorHandler = - event.errorhandler ?? - ((error) => { - logger.error(error.message); - }); - errorHandler(error); - } - }); - } - - get events(): EventNames[] { - return this.emitter.eventNames() as EventNames[]; - } - - protected bindEvents(events: EmitterEvent[]): void { - const eventNames = this.emitter.eventNames(); - for (const event of events) { - if (!eventNames.includes(event.name)) { - this.eventsMap.set(event.name, event); - this.emitter.on(event.name, event.handler(this.storageEngine).bind(event)); - } else { - logger.warn(`Event ${event.name} is already registered. Skipping.`); - } - } - } - - protected emit(eventName: EventNames, data: EmitterEventData): void { - if (!this.emitter.eventNames().includes(eventName)) { - throw new Error(`Event ${eventName} is not registered.`); - } - this.emitter.emit(eventName, data); - } - - protected removeListeners(eventName?: EventNames): void { - if (eventName) { - this.emitter.removeAllListeners(eventName); - logger.info(`Removed all listeners for event: ${eventName}`); - } else { - this.stop(); - } - } - - protected stop(): void { - this.emitter.removeAllListeners(); - logger.info('Emitter engine shut down and all listeners removed.'); - } -} export class EmitterEngine extends AbstractEmitterEngine implements BaseEmitterEngine { + private active: boolean; constructor(events: EmitterEvent[], storageRef: storage.StorageEngine) { super(storageRef); + this.active = process.env.MICRO_SERVICE_NAME === 'powersync'; + logger.info(`EmitterEngine initialized with active status: ${this.active}`); this.bindEvents(events); } @@ -87,7 +30,9 @@ export class EmitterEngine extends AbstractEmitterEngine implements BaseEmitterE } emitEvent(eventName: EventNames, data: EmitterEventData): void { - return this.emit(eventName, data); + if (this.active) { + return this.emit(eventName, data); + } } async shutDown(): Promise { From fea7be8fd5a7192864e4b97ae3392791ad26cebc Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 14 Jul 2025 15:10:14 +0200 Subject: [PATCH 003/169] chnages based on feedback --- .../development_packages_release.yaml | 2 +- .../src/emitters/AbstractEmitterEngine.ts | 64 ------------------ .../src/emitters/EmitterEngine.ts | 62 +++++++++++------- .../src/emitters/emitter-interfaces.ts | 65 +++++++------------ .../service-core/src/emitters/event-error.ts | 10 --- .../src/emitters/events/connect-event.ts | 34 ---------- .../src/emitters/events/disconnect-event.ts | 36 ---------- .../service-core/src/emitters/events/index.ts | 4 -- .../src/routes/endpoints/sync-stream.ts | 13 ++-- .../service-core/src/system/ServiceContext.ts | 3 +- packages/types/src/events.ts | 10 +++ packages/types/src/index.ts | 1 + 12 files changed, 78 insertions(+), 226 deletions(-) delete mode 100644 packages/service-core/src/emitters/AbstractEmitterEngine.ts delete mode 100644 packages/service-core/src/emitters/event-error.ts delete mode 100644 packages/service-core/src/emitters/events/connect-event.ts delete mode 100644 packages/service-core/src/emitters/events/disconnect-event.ts delete mode 100644 packages/service-core/src/emitters/events/index.ts create mode 100644 packages/types/src/events.ts diff --git a/.github/workflows/development_packages_release.yaml b/.github/workflows/development_packages_release.yaml index 0081557dc..2ae4fb7a7 100644 --- a/.github/workflows/development_packages_release.yaml +++ b/.github/workflows/development_packages_release.yaml @@ -6,7 +6,7 @@ on: workflow_dispatch jobs: publish_packages: - name: Publish Devevelopment Packages + name: Publish Development Packages runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/packages/service-core/src/emitters/AbstractEmitterEngine.ts b/packages/service-core/src/emitters/AbstractEmitterEngine.ts deleted file mode 100644 index e1e20b301..000000000 --- a/packages/service-core/src/emitters/AbstractEmitterEngine.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { EmitterEvent, EmitterEventData, EventNames, isEventError } from './emitter-interfaces.js'; -import * as storage from '../storage/storage-index.js'; -import EventEmitter from 'node:events'; -import { logger } from '@powersync/lib-services-framework'; -import { EventError } from './event-error.js'; - -export abstract class AbstractEmitterEngine { - private emitter: EventEmitter; - storageEngine: storage.StorageEngine; - eventsMap: Map = new Map(); - - protected constructor(storage: storage.StorageEngine) { - this.emitter = new EventEmitter({ captureRejections: true }); - this.storageEngine = storage; - this.emitter.on('error', (error: Error | EventError) => { - if (isEventError(error) && this.eventsMap.has(error.eventName)) { - const event = this.eventsMap.get(error.eventName)!; - const errorHandler = - event.errorhandler ?? - ((error) => { - logger.error(error.message); - }); - errorHandler(error); - } - }); - } - - get events(): EventNames[] { - return this.emitter.eventNames() as EventNames[]; - } - - protected bindEvents(events: EmitterEvent[]): void { - const eventNames = this.emitter.eventNames(); - for (const event of events) { - if (!eventNames.includes(event.name)) { - this.eventsMap.set(event.name, event); - this.emitter.on(event.name, event.handler(this.storageEngine).bind(event)); - } else { - logger.warn(`Event ${event.name} is already registered. Skipping.`); - } - } - } - - protected emit(eventName: EventNames, data: EmitterEventData): void { - if (!this.emitter.eventNames().includes(eventName)) { - throw new Error(`Event ${eventName} is not registered.`); - } - this.emitter.emit(eventName, data); - } - - protected removeListeners(eventName?: EventNames): void { - if (eventName) { - this.emitter.removeAllListeners(eventName); - logger.info(`Removed all listeners for event: ${eventName}`); - } else { - this.stop(); - } - } - - protected stop(): void { - this.emitter.removeAllListeners(); - logger.info('Emitter engine shut down and all listeners removed.'); - } -} diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 32632925e..700233a7b 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -1,41 +1,57 @@ -import { BaseEmitterEngine, EmitterEvent, EmitterEventData, EventNames } from './emitter-interfaces.js'; -import * as storage from '../storage/storage-index.js'; -import { AbstractEmitterEngine } from './AbstractEmitterEngine.js'; +import { BaseEmitterEngine } from './emitter-interfaces.js'; +import EventEmitter from 'node:events'; import { logger } from '@powersync/lib-services-framework'; +import { event_types } from '@powersync/service-types'; -export class EmitterEngine extends AbstractEmitterEngine implements BaseEmitterEngine { - private active: boolean; - constructor(events: EmitterEvent[], storageRef: storage.StorageEngine) { - super(storageRef); +export class EmitterEngine implements BaseEmitterEngine { + private emitter: EventEmitter; + eventsMap: Map = new Map(); + private readonly active: boolean; + + constructor() { this.active = process.env.MICRO_SERVICE_NAME === 'powersync'; - logger.info(`EmitterEngine initialized with active status: ${this.active}`); - this.bindEvents(events); + this.emitter = new EventEmitter({ captureRejections: true }); + this.emitter.on('error', (error: Error) => { + logger.error(error.message); + }); + } + + eventNames(): event_types.EmitterEngineEventNames[] { + return this.emitter.eventNames() as event_types.EmitterEngineEventNames[]; } - event(eventName: EventNames): EmitterEvent { - const event = this.eventsMap.get(eventName); - if (!event) { + getStoredEvent(eventName: event_types.EmitterEngineEventNames): event_types.EmitterEvent { + if (!this.eventsMap.has(eventName)) { throw new Error(`Event ${eventName} is not registered.`); } - return event; + return this.eventsMap.get(eventName) as event_types.EmitterEvent; } - removeListeners(eventName?: EventNames): void { - if (eventName) { - this.removeListeners(eventName); - } + get events(): event_types.EmitterEngineEventNames[] { + return this.emitter.eventNames() as event_types.EmitterEngineEventNames[]; } - eventNames(): EventNames[] { - return this.events; + + bindEvent(event: event_types.EmitterEvent): void { + const eventNames = this.emitter.eventNames(); + if (!eventNames.includes(event.name)) { + this.eventsMap.set(event.name, event); + this.emitter.on(event.name, event.handler.bind(event)); + } else { + logger.warn(`Event ${event.name} is already registered. Skipping.`); + } } - emitEvent(eventName: EventNames, data: EmitterEventData): void { + emitEvent(eventName: event_types.EmitterEngineEventNames, data: any): void { + if (!this.emitter.eventNames().includes(eventName)) { + throw new Error(`Event ${eventName} is not registered.`); + } if (this.active) { - return this.emit(eventName, data); + this.emitter.emit(eventName, { ...data, type: eventName }); } } - async shutDown(): Promise { - this.stop(); + shutDown(): void { + this.emitter.removeAllListeners(); + logger.info('Emitter engine shut down and all listeners removed.'); } } diff --git a/packages/service-core/src/emitters/emitter-interfaces.ts b/packages/service-core/src/emitters/emitter-interfaces.ts index 7ab57edf7..b0bad86e2 100644 --- a/packages/service-core/src/emitters/emitter-interfaces.ts +++ b/packages/service-core/src/emitters/emitter-interfaces.ts @@ -1,47 +1,26 @@ -import { JwtPayload } from '../auth/JwtPayload.js'; -import * as storage from '../storage/storage-index.js'; -import { EventError } from './event-error.js'; +// export type SdkEvent = { +// client_id?: string; +// user_id: string; +// user_agent: string; +// jwt_token?: JwtPayload; +// }; +// +// export type EventConnectData = { +// type: EmitterEngineEventNames.SDK_CONNECT_EVENT; +// connect_at: number; +// } & SdkEvent; +// export type EventDisconnectData = { +// type: EmitterEngineEventNames.SDK_DISCONNECT_EVENT; +// disconnect_at: number; +// } & SdkEvent; -export enum EventNames { - // SdK events - SDK_CONNECT_EVENT = 'sdk-connect-event', - SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' -} - -export type SdkEvent = { - client_id?: string; - user_id: string; - user_agent: string; - jwt_token?: JwtPayload; -}; - -export type EventConnectData = { - type: EventNames.SDK_CONNECT_EVENT; - connect_at: number; -} & SdkEvent; - -export type EventDisconnectData = { - type: EventNames.SDK_DISCONNECT_EVENT; - disconnect_at: number; -} & SdkEvent; - -export type EmitterEventData = EventConnectData | EventDisconnectData; -export type EventHandler = (data: EmitterEventData) => Promise; -export type StorageHandler = (storageEngine: storage.StorageEngine) => EventHandler; -export interface EmitterEvent { - name: EventNames; - handler: StorageHandler; - errorhandler?: (error: Error | EventError) => void; -} +import { event_types } from '@powersync/service-types'; export interface BaseEmitterEngine { - eventNames(): EventNames[]; - emitEvent(eventName: EventNames, data: EmitterEventData): void; - removeListeners(eventName?: EventNames): void; - event(eventName: EventNames): EmitterEvent; - shutDown(): Promise; -} - -export function isEventError(error: Error | EventError): error is EventError { - return (error as EventError).eventName !== undefined; + events: event_types.EmitterEngineEventNames[]; + bindEvent(events: event_types.EmitterEvent): void; + eventNames(): event_types.EmitterEngineEventNames[]; + emitEvent(eventName: event_types.EmitterEngineEventNames, data: any): void; + getStoredEvent(eventName: event_types.EmitterEngineEventNames): event_types.EmitterEvent; + shutDown(): void; } diff --git a/packages/service-core/src/emitters/event-error.ts b/packages/service-core/src/emitters/event-error.ts deleted file mode 100644 index 438f13cf8..000000000 --- a/packages/service-core/src/emitters/event-error.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { EventNames } from './emitter-interfaces.js'; - -export class EventError extends Error { - constructor( - public eventName: EventNames, - message: string - ) { - super(message); - } -} diff --git a/packages/service-core/src/emitters/events/connect-event.ts b/packages/service-core/src/emitters/events/connect-event.ts deleted file mode 100644 index c99c9fbbd..000000000 --- a/packages/service-core/src/emitters/events/connect-event.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { EmitterEvent, EmitterEventData, EventConnectData, EventHandler, EventNames } from '../emitter-interfaces.js'; -import * as storage from '../../storage/storage-index.js'; -import { EventError } from '../event-error.js'; -import { logger } from '@powersync/lib-services-framework'; - -export class ConnectEvent implements EmitterEvent { - private type = EventNames.SDK_CONNECT_EVENT; - get name(): EventNames { - return this.type; - } - errorhandler(error: Error | EventError): void { - if (error instanceof EventError) { - logger.error(`EventError in ${error.eventName}:`, error.message); - } else { - logger.error(error.message); - } - } - handler(storageEngine: storage.StorageEngine): EventHandler { - // TODO: USE STORAGE ENGINE - const storage = storageEngine.activeStorage.storage; - return async (data: EmitterEventData) => { - try { - const disconnectData = data as EventConnectData; - console.log( - `Connect event triggered for user: ${disconnectData.user_id} at ${new Date(disconnectData.connect_at).toISOString()}` - ); - } catch (error) { - throw new EventError(this.type, error.message); - } - }; - } -} - -export const connectEvent = new ConnectEvent(); diff --git a/packages/service-core/src/emitters/events/disconnect-event.ts b/packages/service-core/src/emitters/events/disconnect-event.ts deleted file mode 100644 index 2060f6bfd..000000000 --- a/packages/service-core/src/emitters/events/disconnect-event.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { - EmitterEvent, - EmitterEventData, - EventDisconnectData, - EventHandler, - EventNames -} from '../emitter-interfaces.js'; -import * as storage from '../../storage/storage-index.js'; -import { EventError } from '../event-error.js'; -import { logger } from '@powersync/lib-services-framework'; - -export class DisconnectEvent implements EmitterEvent { - private type = EventNames.SDK_DISCONNECT_EVENT; - get name(): EventNames { - return this.type; - } - errorhandler(error: Error | EventError): void { - if (error instanceof EventError) { - logger.error(`EventError in ${error.eventName}:`, error.message); - } else { - logger.error(error.message); - } - } - handler(storageEngine: storage.StorageEngine): EventHandler { - // TODO: USE STORAGE ENGINE - const storage = storageEngine.activeStorage.storage; - return async (data: EmitterEventData) => { - const disconnectData = data as EventDisconnectData; - console.log( - `Disconnect event triggered for user: ${disconnectData.user_id} at ${new Date(disconnectData.disconnect_at).toISOString()}` - ); - }; - } -} - -export const disconnectEvent = new DisconnectEvent(); diff --git a/packages/service-core/src/emitters/events/index.ts b/packages/service-core/src/emitters/events/index.ts deleted file mode 100644 index a2b57baff..000000000 --- a/packages/service-core/src/emitters/events/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { disconnectEvent } from './disconnect-event.js'; -import { connectEvent } from './connect-event.js'; - -export const events = [disconnectEvent, connectEvent]; diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 22170da87..621ec348d 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -7,9 +7,7 @@ import * as util from '../../util/util-index.js'; import { authUser } from '../auth.js'; import { routeDefinition } from '../router.js'; - -import { APIMetric } from '@powersync/service-types'; -import { EventNames } from '../../emitters/emitter-interfaces.js'; +import { APIMetric, event_types } from '@powersync/service-types'; export enum SyncRoutes { STREAM = '/sync/stream' @@ -69,8 +67,7 @@ export const syncStreamed = routeDefinition({ const tracker = new sync.RequestTracker(metricsEngine); try { metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); - service_context.emitterEngine.emitEvent(EventNames.SDK_CONNECT_EVENT, { - type: EventNames.SDK_CONNECT_EVENT, + service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_CONNECT_EVENT, { ...sdkData, connect_at: streamStart }); @@ -136,8 +133,7 @@ export const syncStreamed = routeDefinition({ } controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emitEvent(EventNames.SDK_DISCONNECT_EVENT, { - type: EventNames.SDK_DISCONNECT_EVENT, + service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_DISCONNECT_EVENT, { ...sdkData, disconnect_at: Date.now() }); @@ -151,8 +147,7 @@ export const syncStreamed = routeDefinition({ } catch (ex) { controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emitEvent(EventNames.SDK_DISCONNECT_EVENT, { - type: EventNames.SDK_DISCONNECT_EVENT, + service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_DISCONNECT_EVENT, { ...sdkData, disconnect_at: Date.now() }); diff --git a/packages/service-core/src/system/ServiceContext.ts b/packages/service-core/src/system/ServiceContext.ts index 830daadef..973e4d833 100644 --- a/packages/service-core/src/system/ServiceContext.ts +++ b/packages/service-core/src/system/ServiceContext.ts @@ -9,7 +9,6 @@ import * as storage from '../storage/storage-index.js'; import { SyncContext } from '../sync/SyncContext.js'; import * as utils from '../util/util-index.js'; import { EmitterEngine } from '../emitters/EmitterEngine.js'; -import { events } from '../emitters/events/index.js'; export interface ServiceContext { configuration: utils.ResolvedPowerSyncConfig; @@ -70,7 +69,7 @@ export class ServiceContextContainer implements ServiceContext { } }); - this.emitterEngine = new EmitterEngine(events, this.storageEngine); + this.emitterEngine = new EmitterEngine(); this.lifeCycleEngine.withLifecycle(this.storageEngine, { start: (storageEngine) => storageEngine.start(), diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts new file mode 100644 index 000000000..4f5f23cf9 --- /dev/null +++ b/packages/types/src/events.ts @@ -0,0 +1,10 @@ +export enum EmitterEngineEventNames { + SDK_CONNECT_EVENT = 'sdk-connect-event', + SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' +} + +export type EventHandlerFunc = (data: any) => Promise; +export interface EmitterEvent { + name: EmitterEngineEventNames; + handler: EventHandlerFunc; +} diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index dca03e082..1f600c259 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -4,3 +4,4 @@ export * from './definitions.js'; export * as internal_routes from './routes.js'; export * from './metrics.js'; export * as metric_types from './metrics.js'; +export * as event_types from './events.js'; From 0f603cb524f3c03daaf1397c82a3c81657160917 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 14 Jul 2025 15:26:34 +0200 Subject: [PATCH 004/169] refactoring and some renaming --- packages/service-core/src/emitters/EmitterEngine.ts | 12 ++++-------- .../service-core/src/emitters/emitter-interfaces.ts | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 700233a7b..fc28d138b 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -6,13 +6,10 @@ import { event_types } from '@powersync/service-types'; export class EmitterEngine implements BaseEmitterEngine { private emitter: EventEmitter; eventsMap: Map = new Map(); - private readonly active: boolean; - constructor() { - this.active = process.env.MICRO_SERVICE_NAME === 'powersync'; this.emitter = new EventEmitter({ captureRejections: true }); this.emitter.on('error', (error: Error) => { - logger.error(error.message); + logger.error(error); }); } @@ -27,7 +24,7 @@ export class EmitterEngine implements BaseEmitterEngine { return this.eventsMap.get(eventName) as event_types.EmitterEvent; } - get events(): event_types.EmitterEngineEventNames[] { + get listEvents(): event_types.EmitterEngineEventNames[] { return this.emitter.eventNames() as event_types.EmitterEngineEventNames[]; } @@ -43,9 +40,8 @@ export class EmitterEngine implements BaseEmitterEngine { emitEvent(eventName: event_types.EmitterEngineEventNames, data: any): void { if (!this.emitter.eventNames().includes(eventName)) { - throw new Error(`Event ${eventName} is not registered.`); - } - if (this.active) { + logger.error(`Event ${eventName} is not registered.`); + } else { this.emitter.emit(eventName, { ...data, type: eventName }); } } diff --git a/packages/service-core/src/emitters/emitter-interfaces.ts b/packages/service-core/src/emitters/emitter-interfaces.ts index b0bad86e2..ac49b65fe 100644 --- a/packages/service-core/src/emitters/emitter-interfaces.ts +++ b/packages/service-core/src/emitters/emitter-interfaces.ts @@ -17,7 +17,7 @@ import { event_types } from '@powersync/service-types'; export interface BaseEmitterEngine { - events: event_types.EmitterEngineEventNames[]; + listEvents: event_types.EmitterEngineEventNames[]; bindEvent(events: event_types.EmitterEvent): void; eventNames(): event_types.EmitterEngineEventNames[]; emitEvent(eventName: event_types.EmitterEngineEventNames, data: any): void; From ef6cfd11a282790055a8f299a1be961001550e1c Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 14 Jul 2025 15:40:15 +0200 Subject: [PATCH 005/169] controller addition handler --- packages/types/src/events.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 4f5f23cf9..30981c343 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -3,7 +3,7 @@ export enum EmitterEngineEventNames { SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' } -export type EventHandlerFunc = (data: any) => Promise; +export type EventHandlerFunc = (data: any) => Promise | ((controller: any) => (data: any) => void); export interface EmitterEvent { name: EmitterEngineEventNames; handler: EventHandlerFunc; From 65c2f5b6f5602e5508e4e087de54b837528a06a7 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 14 Jul 2025 15:56:09 +0200 Subject: [PATCH 006/169] type fix --- packages/types/src/events.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 30981c343..f309d8a04 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -3,7 +3,9 @@ export enum EmitterEngineEventNames { SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' } -export type EventHandlerFunc = (data: any) => Promise | ((controller: any) => (data: any) => void); +export type EventHandlerFunc = + | ((data: any) => Promise | void) + | ((controller: any) => (data: any) => Promise | void); export interface EmitterEvent { name: EmitterEngineEventNames; handler: EventHandlerFunc; From 777d42637f623563f3dbfa91d785c6c0514c267e Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 14 Jul 2025 17:10:51 +0200 Subject: [PATCH 007/169] dev packages --- modules/module-core/package.json | 4 ++-- modules/module-mongodb-storage/package.json | 4 ++-- modules/module-mongodb/package.json | 4 ++-- modules/module-mysql/package.json | 4 ++-- modules/module-postgres-storage/package.json | 4 ++-- modules/module-postgres/package.json | 4 ++-- packages/service-core/src/emitters/EmitterEngine.ts | 1 + service/package.json | 2 +- 8 files changed, 14 insertions(+), 13 deletions(-) diff --git a/modules/module-core/package.json b/modules/module-core/package.json index 2b888e248..8991a4322 100644 --- a/modules/module-core/package.json +++ b/modules/module-core/package.json @@ -29,9 +29,9 @@ }, "dependencies": { "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "workspace:*", + "@powersync/service-core": "0.0.0-dev-20250714140103", + "@powersync/service-types": "0.0.0-dev-20250714140103", "@powersync/service-rsocket-router": "workspace:*", - "@powersync/service-types": "workspace:*", "fastify": "4.23.2", "@fastify/cors": "8.4.1" }, diff --git a/modules/module-mongodb-storage/package.json b/modules/module-mongodb-storage/package.json index 9b6aae0c7..fbd0b0b56 100644 --- a/modules/module-mongodb-storage/package.json +++ b/modules/module-mongodb-storage/package.json @@ -30,10 +30,10 @@ "dependencies": { "@powersync/lib-service-mongodb": "workspace:*", "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "workspace:*", + "@powersync/service-core": "0.0.0-dev-20250714140103", + "@powersync/service-types": "0.0.0-dev-20250714140103", "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", - "@powersync/service-types": "workspace:*", "bson": "^6.10.3", "ix": "^5.0.0", "lru-cache": "^10.2.2", diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index 59bfcf485..750996703 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -30,10 +30,10 @@ "dependencies": { "@powersync/lib-service-mongodb": "workspace:*", "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "workspace:*", + "@powersync/service-core": "0.0.0-dev-20250714140103", + "@powersync/service-types": "0.0.0-dev-20250714140103", "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", - "@powersync/service-types": "workspace:*", "bson": "^6.10.3", "ts-codec": "^1.3.0", "uuid": "^11.1.0" diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index a31701268..053c20245 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -29,9 +29,9 @@ }, "dependencies": { "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "workspace:*", + "@powersync/service-core": "0.0.0-dev-20250714140103", + "@powersync/service-types": "0.0.0-dev-20250714140103", "@powersync/service-sync-rules": "workspace:*", - "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/mysql-zongji": "0.2.0", "async": "^3.2.4", diff --git a/modules/module-postgres-storage/package.json b/modules/module-postgres-storage/package.json index 04cff7a29..28446c162 100644 --- a/modules/module-postgres-storage/package.json +++ b/modules/module-postgres-storage/package.json @@ -31,12 +31,12 @@ "dependencies": { "@powersync/lib-service-postgres": "workspace:*", "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "workspace:*", + "@powersync/service-core": "0.0.0-dev-20250714140103", + "@powersync/service-types": "0.0.0-dev-20250714140103", "@powersync/service-core-tests": "workspace:*", "@powersync/service-jpgwire": "workspace:*", "@powersync/service-jsonbig": "^0.17.10", "@powersync/service-sync-rules": "workspace:*", - "@powersync/service-types": "workspace:*", "ix": "^5.0.0", "lru-cache": "^10.2.2", "p-defer": "^4.0.1", diff --git a/modules/module-postgres/package.json b/modules/module-postgres/package.json index 7bcc017fa..f48531e68 100644 --- a/modules/module-postgres/package.json +++ b/modules/module-postgres/package.json @@ -30,11 +30,11 @@ "dependencies": { "@powersync/lib-service-postgres": "workspace:*", "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "workspace:*", + "@powersync/service-core": "0.0.0-dev-20250714140103", + "@powersync/service-types": "0.0.0-dev-20250714140103", "@powersync/service-jpgwire": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", - "@powersync/service-types": "workspace:*", "jose": "^4.15.1", "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87", "semver": "^7.5.4", diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index fc28d138b..2e63aae0d 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -31,6 +31,7 @@ export class EmitterEngine implements BaseEmitterEngine { bindEvent(event: event_types.EmitterEvent): void { const eventNames = this.emitter.eventNames(); if (!eventNames.includes(event.name)) { + logger.info('Registering event:', event.name); this.eventsMap.set(event.name, event); this.emitter.on(event.name, event.handler.bind(event)); } else { diff --git a/service/package.json b/service/package.json index 6fd0f7f9f..43da448be 100644 --- a/service/package.json +++ b/service/package.json @@ -10,7 +10,7 @@ "clean": "rm -rf ./lib && tsc -b --clean" }, "dependencies": { - "@powersync/service-core": "workspace:*", + "@powersync/service-core": "0.0.0-dev-20250714140103", "@powersync/lib-services-framework": "workspace:*", "@powersync/service-module-postgres": "workspace:*", "@powersync/service-module-postgres-storage": "workspace:*", From f6370a6dde90eaf3bd11f69be58bedc2e1faef98 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 14 Jul 2025 17:13:06 +0200 Subject: [PATCH 008/169] forgot lockfile --- pnpm-lock.yaml | 148 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 122 insertions(+), 26 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 943acc6b8..79731994c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -160,14 +160,14 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/service-core': - specifier: workspace:* - version: link:../../packages/service-core + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 '@powersync/service-rsocket-router': specifier: workspace:* version: link:../../packages/rsocket-router '@powersync/service-types': - specifier: workspace:* - version: link:../../packages/types + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 fastify: specifier: 4.23.2 version: 4.23.2 @@ -181,8 +181,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/service-core': - specifier: workspace:* - version: link:../../packages/service-core + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 '@powersync/service-jsonbig': specifier: workspace:* version: link:../../packages/jsonbig @@ -190,8 +190,8 @@ importers: specifier: workspace:* version: link:../../packages/sync-rules '@powersync/service-types': - specifier: workspace:* - version: link:../../packages/types + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 bson: specifier: ^6.10.3 version: 6.10.3 @@ -221,8 +221,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/service-core': - specifier: workspace:* - version: link:../../packages/service-core + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 '@powersync/service-jsonbig': specifier: workspace:* version: link:../../packages/jsonbig @@ -230,8 +230,8 @@ importers: specifier: workspace:* version: link:../../packages/sync-rules '@powersync/service-types': - specifier: workspace:* - version: link:../../packages/types + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 bson: specifier: ^6.10.3 version: 6.10.3 @@ -261,8 +261,8 @@ importers: specifier: 0.2.0 version: 0.2.0 '@powersync/service-core': - specifier: workspace:* - version: link:../../packages/service-core + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 '@powersync/service-jsonbig': specifier: workspace:* version: link:../../packages/jsonbig @@ -270,8 +270,8 @@ importers: specifier: workspace:* version: link:../../packages/sync-rules '@powersync/service-types': - specifier: workspace:* - version: link:../../packages/types + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 async: specifier: ^3.2.4 version: 3.2.5 @@ -316,8 +316,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/service-core': - specifier: workspace:* - version: link:../../packages/service-core + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 '@powersync/service-jpgwire': specifier: workspace:* version: link:../../packages/jpgwire @@ -328,8 +328,8 @@ importers: specifier: workspace:* version: link:../../packages/sync-rules '@powersync/service-types': - specifier: workspace:* - version: link:../../packages/types + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 jose: specifier: ^4.15.1 version: 4.15.9 @@ -371,8 +371,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/service-core': - specifier: workspace:* - version: link:../../packages/service-core + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 '@powersync/service-core-tests': specifier: workspace:* version: link:../../packages/service-core-tests @@ -386,8 +386,8 @@ importers: specifier: workspace:* version: link:../../packages/sync-rules '@powersync/service-types': - specifier: workspace:* - version: link:../../packages/types + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 ix: specifier: ^5.0.0 version: 5.0.0 @@ -648,8 +648,8 @@ importers: specifier: workspace:* version: link:../libs/lib-services '@powersync/service-core': - specifier: workspace:* - version: link:../packages/service-core + specifier: 0.0.0-dev-20250714140103 + version: 0.0.0-dev-20250714140103 '@powersync/service-module-core': specifier: workspace:* version: link:../modules/module-core @@ -1281,13 +1281,31 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} + '@powersync/lib-services-framework@0.7.0': + resolution: {integrity: sha512-4e7341gqiXXEdx7kOr4z/6V1e0mlzDdYyJLANif00AqyAl4QIMPXPRVRoiVBprEEIecbXlKqwXEG6EuPCL/6AA==} + '@powersync/mysql-zongji@0.2.0': resolution: {integrity: sha512-ua/n7WFfoiXmqfgwLikcm/AaDE6+t5gFVTWHWsbiuRQMNtXE1F2gXpZJdwKhr8WsOCYkB/A1ZOgbJKi4tK342g==} engines: {node: '>=22.0.0'} + '@powersync/service-core@0.0.0-dev-20250714140103': + resolution: {integrity: sha512-cUGnv2+CESqvyTwbVPcu05L/LUhhV24MWZXMeux17ACJruOaiNJL+u+Ww7dHfjmq+sjQINRqcENlpI3cbse0Mw==} + + '@powersync/service-errors@0.3.1': + resolution: {integrity: sha512-KK7H6pm7g9Jv50L+BVchmAIX+xBNYXwoysYs9v5E0zb3jClzCvRQXoB70/x+vQgIOuFyfgAPAluNgWb7dEvW+w==} + '@powersync/service-jsonbig@0.17.10': resolution: {integrity: sha512-BgxgUewuw4HFCM9MzuzlIuRKHya6rimNPYqUItt7CO3ySUeUnX8Qn9eZpMxu9AT5Y8zqkSyxvduY36zZueNojg==} + '@powersync/service-rsocket-router@0.1.1': + resolution: {integrity: sha512-x1lnXerJEEbZ/6rXiXkjv43rbp4kbecZf24g9ZOdsmF2FQ76SIg8vlr8pFwVLsjfZpz16h/dxzeFUxdydSSG1A==} + + '@powersync/service-sync-rules@0.27.0': + resolution: {integrity: sha512-CR2WIjcK+UhlfIRD+2ZX815pxPaSxKbMaBPfRgaROZ9/avf410/pEEVJoHBl9MYmXrcyB3GWkS9MpNd1Led5Kg==} + + '@powersync/service-types@0.0.0-dev-20250714140103': + resolution: {integrity: sha512-AfLVirEZkXogCPbhe4H9eUEZaJypCZmV4G+BxPjYdrqatnz2RDbK1CyM3xhDhj+sC8xlgriJ0Akg9qe5lAJuwA==} + '@prisma/instrumentation@5.16.1': resolution: {integrity: sha512-4m5gRFWnQb8s/yTyGbMZkL7A5uJgqOWcWJxapwcAD0T0kh5sGPEVSQl/zTQvE9aduXhFAxOtC3gO+R8Hb5xO1Q==} @@ -4742,16 +4760,94 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 + '@powersync/lib-services-framework@0.7.0': + dependencies: + '@powersync/service-errors': 0.3.1 + ajv: 8.16.0 + better-ajv-errors: 1.2.0(ajv@8.16.0) + bson: 6.10.3 + dotenv: 16.4.5 + ipaddr.js: 2.2.0 + lodash: 4.17.21 + ts-codec: 1.3.0 + uuid: 11.1.0 + winston: 3.13.1 + zod: 3.23.8 + '@powersync/mysql-zongji@0.2.0': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.52 iconv-lite: 0.6.3 + '@powersync/service-core@0.0.0-dev-20250714140103': + dependencies: + '@js-sdsl/ordered-set': 4.4.2 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/exporter-metrics-otlp-http': 0.57.2(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-prometheus': 0.57.2(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.0) + '@powersync/lib-services-framework': 0.7.0 + '@powersync/service-jsonbig': 0.17.10 + '@powersync/service-rsocket-router': 0.1.1 + '@powersync/service-sync-rules': 0.27.0 + '@powersync/service-types': 0.0.0-dev-20250714140103 + async: 3.2.5 + async-mutex: 0.5.0 + bson: 6.10.3 + commander: 12.1.0 + cors: 2.8.5 + ipaddr.js: 2.2.0 + ix: 5.0.0 + jose: 4.15.9 + lodash: 4.17.21 + lru-cache: 10.4.3 + node-fetch: 3.3.2 + ts-codec: 1.3.0 + uri-js: 4.4.1 + uuid: 11.1.0 + winston: 3.13.1 + yaml: 2.5.0 + transitivePeerDependencies: + - babel-plugin-macros + - bufferutil + - utf-8-validate + + '@powersync/service-errors@0.3.1': {} + '@powersync/service-jsonbig@0.17.10': dependencies: lossless-json: 2.0.11 + '@powersync/service-rsocket-router@0.1.1': + dependencies: + '@powersync/lib-services-framework': 0.7.0 + rsocket-core: 1.0.0-alpha.3 + ts-codec: 1.3.0 + uuid: 11.1.0 + ws: 8.18.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@powersync/service-sync-rules@0.27.0': + dependencies: + '@powersync/service-jsonbig': 0.17.10 + '@syncpoint/wkx': 0.5.2 + ajv: 8.16.0 + pgsql-ast-parser: 11.2.0 + uuid: 11.1.0 + yaml: 2.5.0 + + '@powersync/service-types@0.0.0-dev-20250714140103': + dependencies: + dedent: 1.6.0 + ts-codec: 1.3.0 + uri-js: 4.4.1 + transitivePeerDependencies: + - babel-plugin-macros + '@prisma/instrumentation@5.16.1': dependencies: '@opentelemetry/api': 1.9.0 From 79c9a5d1da0ab47bcf64530b1c1f7202fd228c09 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 15 Jul 2025 07:52:26 +0200 Subject: [PATCH 009/169] reverted dev packages --- modules/module-core/package.json | 4 ++-- modules/module-mongodb-storage/package.json | 4 ++-- modules/module-mongodb/package.json | 4 ++-- modules/module-mysql/package.json | 4 ++-- modules/module-postgres-storage/package.json | 4 ++-- modules/module-postgres/package.json | 4 ++-- service/package.json | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/modules/module-core/package.json b/modules/module-core/package.json index 8991a4322..e6d9e69d5 100644 --- a/modules/module-core/package.json +++ b/modules/module-core/package.json @@ -29,8 +29,8 @@ }, "dependencies": { "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "0.0.0-dev-20250714140103", - "@powersync/service-types": "0.0.0-dev-20250714140103", + "@powersync/service-core": "workspace:*", + "@powersync/service-types": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", "fastify": "4.23.2", "@fastify/cors": "8.4.1" diff --git a/modules/module-mongodb-storage/package.json b/modules/module-mongodb-storage/package.json index fbd0b0b56..3c94d2f50 100644 --- a/modules/module-mongodb-storage/package.json +++ b/modules/module-mongodb-storage/package.json @@ -30,8 +30,8 @@ "dependencies": { "@powersync/lib-service-mongodb": "workspace:*", "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "0.0.0-dev-20250714140103", - "@powersync/service-types": "0.0.0-dev-20250714140103", + "@powersync/service-core": "workspace:*", + "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", "bson": "^6.10.3", diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index 750996703..913f4d5eb 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -30,8 +30,8 @@ "dependencies": { "@powersync/lib-service-mongodb": "workspace:*", "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "0.0.0-dev-20250714140103", - "@powersync/service-types": "0.0.0-dev-20250714140103", + "@powersync/service-core": "workspace:*", + "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", "bson": "^6.10.3", diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 053c20245..b614a7beb 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -29,8 +29,8 @@ }, "dependencies": { "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "0.0.0-dev-20250714140103", - "@powersync/service-types": "0.0.0-dev-20250714140103", + "@powersync/service-core": "workspace:*", + "@powersync/service-types": "workspace:*", "@powersync/service-sync-rules": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/mysql-zongji": "0.2.0", diff --git a/modules/module-postgres-storage/package.json b/modules/module-postgres-storage/package.json index 28446c162..1e1ef736c 100644 --- a/modules/module-postgres-storage/package.json +++ b/modules/module-postgres-storage/package.json @@ -31,8 +31,8 @@ "dependencies": { "@powersync/lib-service-postgres": "workspace:*", "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "0.0.0-dev-20250714140103", - "@powersync/service-types": "0.0.0-dev-20250714140103", + "@powersync/service-core": "workspace:*", + "@powersync/service-types": "workspace:*", "@powersync/service-core-tests": "workspace:*", "@powersync/service-jpgwire": "workspace:*", "@powersync/service-jsonbig": "^0.17.10", diff --git a/modules/module-postgres/package.json b/modules/module-postgres/package.json index f48531e68..c3c03cc72 100644 --- a/modules/module-postgres/package.json +++ b/modules/module-postgres/package.json @@ -30,8 +30,8 @@ "dependencies": { "@powersync/lib-service-postgres": "workspace:*", "@powersync/lib-services-framework": "workspace:*", - "@powersync/service-core": "0.0.0-dev-20250714140103", - "@powersync/service-types": "0.0.0-dev-20250714140103", + "@powersync/service-core": "workspace:*", + "@powersync/service-types": "workspace:*", "@powersync/service-jpgwire": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", diff --git a/service/package.json b/service/package.json index 43da448be..6fd0f7f9f 100644 --- a/service/package.json +++ b/service/package.json @@ -10,7 +10,7 @@ "clean": "rm -rf ./lib && tsc -b --clean" }, "dependencies": { - "@powersync/service-core": "0.0.0-dev-20250714140103", + "@powersync/service-core": "workspace:*", "@powersync/lib-services-framework": "workspace:*", "@powersync/service-module-postgres": "workspace:*", "@powersync/service-module-postgres-storage": "workspace:*", From 4685dd319ce6a4beae8b4afe341fc3059c648e68 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 15 Jul 2025 07:54:19 +0200 Subject: [PATCH 010/169] lockfile dammit --- pnpm-lock.yaml | 148 +++++++++---------------------------------------- 1 file changed, 26 insertions(+), 122 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 79731994c..943acc6b8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -160,14 +160,14 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/service-core': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/service-core '@powersync/service-rsocket-router': specifier: workspace:* version: link:../../packages/rsocket-router '@powersync/service-types': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/types fastify: specifier: 4.23.2 version: 4.23.2 @@ -181,8 +181,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/service-core': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/service-core '@powersync/service-jsonbig': specifier: workspace:* version: link:../../packages/jsonbig @@ -190,8 +190,8 @@ importers: specifier: workspace:* version: link:../../packages/sync-rules '@powersync/service-types': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/types bson: specifier: ^6.10.3 version: 6.10.3 @@ -221,8 +221,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/service-core': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/service-core '@powersync/service-jsonbig': specifier: workspace:* version: link:../../packages/jsonbig @@ -230,8 +230,8 @@ importers: specifier: workspace:* version: link:../../packages/sync-rules '@powersync/service-types': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/types bson: specifier: ^6.10.3 version: 6.10.3 @@ -261,8 +261,8 @@ importers: specifier: 0.2.0 version: 0.2.0 '@powersync/service-core': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/service-core '@powersync/service-jsonbig': specifier: workspace:* version: link:../../packages/jsonbig @@ -270,8 +270,8 @@ importers: specifier: workspace:* version: link:../../packages/sync-rules '@powersync/service-types': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/types async: specifier: ^3.2.4 version: 3.2.5 @@ -316,8 +316,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/service-core': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/service-core '@powersync/service-jpgwire': specifier: workspace:* version: link:../../packages/jpgwire @@ -328,8 +328,8 @@ importers: specifier: workspace:* version: link:../../packages/sync-rules '@powersync/service-types': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/types jose: specifier: ^4.15.1 version: 4.15.9 @@ -371,8 +371,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/service-core': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/service-core '@powersync/service-core-tests': specifier: workspace:* version: link:../../packages/service-core-tests @@ -386,8 +386,8 @@ importers: specifier: workspace:* version: link:../../packages/sync-rules '@powersync/service-types': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../../packages/types ix: specifier: ^5.0.0 version: 5.0.0 @@ -648,8 +648,8 @@ importers: specifier: workspace:* version: link:../libs/lib-services '@powersync/service-core': - specifier: 0.0.0-dev-20250714140103 - version: 0.0.0-dev-20250714140103 + specifier: workspace:* + version: link:../packages/service-core '@powersync/service-module-core': specifier: workspace:* version: link:../modules/module-core @@ -1281,31 +1281,13 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/lib-services-framework@0.7.0': - resolution: {integrity: sha512-4e7341gqiXXEdx7kOr4z/6V1e0mlzDdYyJLANif00AqyAl4QIMPXPRVRoiVBprEEIecbXlKqwXEG6EuPCL/6AA==} - '@powersync/mysql-zongji@0.2.0': resolution: {integrity: sha512-ua/n7WFfoiXmqfgwLikcm/AaDE6+t5gFVTWHWsbiuRQMNtXE1F2gXpZJdwKhr8WsOCYkB/A1ZOgbJKi4tK342g==} engines: {node: '>=22.0.0'} - '@powersync/service-core@0.0.0-dev-20250714140103': - resolution: {integrity: sha512-cUGnv2+CESqvyTwbVPcu05L/LUhhV24MWZXMeux17ACJruOaiNJL+u+Ww7dHfjmq+sjQINRqcENlpI3cbse0Mw==} - - '@powersync/service-errors@0.3.1': - resolution: {integrity: sha512-KK7H6pm7g9Jv50L+BVchmAIX+xBNYXwoysYs9v5E0zb3jClzCvRQXoB70/x+vQgIOuFyfgAPAluNgWb7dEvW+w==} - '@powersync/service-jsonbig@0.17.10': resolution: {integrity: sha512-BgxgUewuw4HFCM9MzuzlIuRKHya6rimNPYqUItt7CO3ySUeUnX8Qn9eZpMxu9AT5Y8zqkSyxvduY36zZueNojg==} - '@powersync/service-rsocket-router@0.1.1': - resolution: {integrity: sha512-x1lnXerJEEbZ/6rXiXkjv43rbp4kbecZf24g9ZOdsmF2FQ76SIg8vlr8pFwVLsjfZpz16h/dxzeFUxdydSSG1A==} - - '@powersync/service-sync-rules@0.27.0': - resolution: {integrity: sha512-CR2WIjcK+UhlfIRD+2ZX815pxPaSxKbMaBPfRgaROZ9/avf410/pEEVJoHBl9MYmXrcyB3GWkS9MpNd1Led5Kg==} - - '@powersync/service-types@0.0.0-dev-20250714140103': - resolution: {integrity: sha512-AfLVirEZkXogCPbhe4H9eUEZaJypCZmV4G+BxPjYdrqatnz2RDbK1CyM3xhDhj+sC8xlgriJ0Akg9qe5lAJuwA==} - '@prisma/instrumentation@5.16.1': resolution: {integrity: sha512-4m5gRFWnQb8s/yTyGbMZkL7A5uJgqOWcWJxapwcAD0T0kh5sGPEVSQl/zTQvE9aduXhFAxOtC3gO+R8Hb5xO1Q==} @@ -4760,94 +4742,16 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/lib-services-framework@0.7.0': - dependencies: - '@powersync/service-errors': 0.3.1 - ajv: 8.16.0 - better-ajv-errors: 1.2.0(ajv@8.16.0) - bson: 6.10.3 - dotenv: 16.4.5 - ipaddr.js: 2.2.0 - lodash: 4.17.21 - ts-codec: 1.3.0 - uuid: 11.1.0 - winston: 3.13.1 - zod: 3.23.8 - '@powersync/mysql-zongji@0.2.0': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.52 iconv-lite: 0.6.3 - '@powersync/service-core@0.0.0-dev-20250714140103': - dependencies: - '@js-sdsl/ordered-set': 4.4.2 - '@opentelemetry/api': 1.9.0 - '@opentelemetry/exporter-metrics-otlp-http': 0.57.2(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-prometheus': 0.57.2(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.0) - '@powersync/lib-services-framework': 0.7.0 - '@powersync/service-jsonbig': 0.17.10 - '@powersync/service-rsocket-router': 0.1.1 - '@powersync/service-sync-rules': 0.27.0 - '@powersync/service-types': 0.0.0-dev-20250714140103 - async: 3.2.5 - async-mutex: 0.5.0 - bson: 6.10.3 - commander: 12.1.0 - cors: 2.8.5 - ipaddr.js: 2.2.0 - ix: 5.0.0 - jose: 4.15.9 - lodash: 4.17.21 - lru-cache: 10.4.3 - node-fetch: 3.3.2 - ts-codec: 1.3.0 - uri-js: 4.4.1 - uuid: 11.1.0 - winston: 3.13.1 - yaml: 2.5.0 - transitivePeerDependencies: - - babel-plugin-macros - - bufferutil - - utf-8-validate - - '@powersync/service-errors@0.3.1': {} - '@powersync/service-jsonbig@0.17.10': dependencies: lossless-json: 2.0.11 - '@powersync/service-rsocket-router@0.1.1': - dependencies: - '@powersync/lib-services-framework': 0.7.0 - rsocket-core: 1.0.0-alpha.3 - ts-codec: 1.3.0 - uuid: 11.1.0 - ws: 8.18.0 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - '@powersync/service-sync-rules@0.27.0': - dependencies: - '@powersync/service-jsonbig': 0.17.10 - '@syncpoint/wkx': 0.5.2 - ajv: 8.16.0 - pgsql-ast-parser: 11.2.0 - uuid: 11.1.0 - yaml: 2.5.0 - - '@powersync/service-types@0.0.0-dev-20250714140103': - dependencies: - dedent: 1.6.0 - ts-codec: 1.3.0 - uri-js: 4.4.1 - transitivePeerDependencies: - - babel-plugin-macros - '@prisma/instrumentation@5.16.1': dependencies: '@opentelemetry/api': 1.9.0 From dd29d5fd4c91af623a72f4c11540f971ce1f9f8b Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 15 Jul 2025 08:44:53 +0200 Subject: [PATCH 011/169] testing --- packages/service-core/src/emitters/EmitterEngine.ts | 1 + packages/types/src/events.ts | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 2e63aae0d..77ddc2f7f 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -43,6 +43,7 @@ export class EmitterEngine implements BaseEmitterEngine { if (!this.emitter.eventNames().includes(eventName)) { logger.error(`Event ${eventName} is not registered.`); } else { + logger.info(`Emitting event: ${eventName}`, data); this.emitter.emit(eventName, { ...data, type: eventName }); } } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index f309d8a04..0fa0e0742 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -3,9 +3,7 @@ export enum EmitterEngineEventNames { SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' } -export type EventHandlerFunc = - | ((data: any) => Promise | void) - | ((controller: any) => (data: any) => Promise | void); +export type EventHandlerFunc = (data: any) => Promise | void; export interface EmitterEvent { name: EmitterEngineEventNames; handler: EventHandlerFunc; From cbdcaaf73bebf9af160e9b578fec33bbb41ec3cc Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 15 Jul 2025 09:12:57 +0200 Subject: [PATCH 012/169] test emitter --- packages/service-core/src/emitters/EmitterEngine.ts | 5 +++-- packages/service-core/src/routes/endpoints/sync-stream.ts | 1 + packages/types/src/events.ts | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 77ddc2f7f..14dde7f56 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -30,10 +30,11 @@ export class EmitterEngine implements BaseEmitterEngine { bindEvent(event: event_types.EmitterEvent): void { const eventNames = this.emitter.eventNames(); + console.log(event); if (!eventNames.includes(event.name)) { - logger.info('Registering event:', event.name); + logger.info(`Registering event: ${event.name}`); this.eventsMap.set(event.name, event); - this.emitter.on(event.name, event.handler.bind(event)); + this.emitter.on(event.name, event.handler); } else { logger.warn(`Event ${event.name} is already registered. Skipping.`); } diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 621ec348d..ca668fd93 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -67,6 +67,7 @@ export const syncStreamed = routeDefinition({ const tracker = new sync.RequestTracker(metricsEngine); try { metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); + console.log(service_context.emitterEngine.getStoredEvent(event_types.EmitterEngineEventNames.SDK_CONNECT_EVENT)); service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_CONNECT_EVENT, { ...sdkData, connect_at: streamStart diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 0fa0e0742..65c85b951 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -6,5 +6,6 @@ export enum EmitterEngineEventNames { export type EventHandlerFunc = (data: any) => Promise | void; export interface EmitterEvent { name: EmitterEngineEventNames; + setController?: (controller: any) => EmitterEvent; handler: EventHandlerFunc; } From fd6b080f4cda1a8ea94f7599432feb3dba8c095a Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 15 Jul 2025 09:27:10 +0200 Subject: [PATCH 013/169] test --- packages/service-core/src/routes/endpoints/sync-stream.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index ca668fd93..a0a16cd3c 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -39,6 +39,11 @@ export const syncStreamed = routeDefinition({ user_agent: userAgent as string, jwt_token: token_payload }; + console.log('\n'); + console.log('DATA: ', JSON.stringify(sdkData, null, 2)); + console.log('\n'); + + console.log('EMITTER ENGINE', service_context.emitterEngine); if (routerEngine.closed) { throw new errors.ServiceError({ @@ -67,7 +72,6 @@ export const syncStreamed = routeDefinition({ const tracker = new sync.RequestTracker(metricsEngine); try { metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); - console.log(service_context.emitterEngine.getStoredEvent(event_types.EmitterEngineEventNames.SDK_CONNECT_EVENT)); service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_CONNECT_EVENT, { ...sdkData, connect_at: streamStart From 2846a618b5e042f40f4755e911333bef674afd34 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 15 Jul 2025 10:10:01 +0200 Subject: [PATCH 014/169] added emiiter to socket route --- .../src/routes/endpoints/socket-route.ts | 18 +++++++++++++++++- .../src/routes/endpoints/sync-stream.ts | 5 ----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 0533aab00..45df161c5 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -7,7 +7,7 @@ import * as util from '../../util/util-index.js'; import { SocketRouteGenerator } from '../router-socket.js'; import { SyncRoutes } from './sync-stream.js'; -import { APIMetric } from '@powersync/service-types'; +import { APIMetric, event_types } from '@powersync/service-types'; export const syncStreamReactive: SocketRouteGenerator = (router) => router.reactiveStream(SyncRoutes.STREAM, { @@ -22,6 +22,14 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => client_id: params.client_id, user_agent: context.user_agent }; + + const sdkData = { + client_id: params.client_id, + user_id: context.user_id!, + user_agent: context.user_agent, + jwt_token: context.token_payload + }; + const streamStart = Date.now(); // Best effort guess on why the stream was closed. @@ -86,6 +94,10 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => }); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); + service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_CONNECT_EVENT, { + ...sdkData, + connect_at: streamStart + }); const tracker = new sync.RequestTracker(metricsEngine); try { for await (const data of sync.streamResponse({ @@ -165,6 +177,10 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => close_reason: closeReason ?? 'unknown' }); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); + service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_DISCONNECT_EVENT, { + ...sdkData, + disconnect_at: Date.now() + }); } } }); diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index a0a16cd3c..621ec348d 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -39,11 +39,6 @@ export const syncStreamed = routeDefinition({ user_agent: userAgent as string, jwt_token: token_payload }; - console.log('\n'); - console.log('DATA: ', JSON.stringify(sdkData, null, 2)); - console.log('\n'); - - console.log('EMITTER ENGINE', service_context.emitterEngine); if (routerEngine.closed) { throw new errors.ServiceError({ From 93bf85a7debcbe153b9a23784d60244711a92142 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 15 Jul 2025 10:42:09 +0200 Subject: [PATCH 015/169] fixed binding --- packages/service-core/src/emitters/EmitterEngine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 14dde7f56..611496811 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -34,7 +34,7 @@ export class EmitterEngine implements BaseEmitterEngine { if (!eventNames.includes(event.name)) { logger.info(`Registering event: ${event.name}`); this.eventsMap.set(event.name, event); - this.emitter.on(event.name, event.handler); + this.emitter.on(event.name, event.handler.bind(event)); } else { logger.warn(`Event ${event.name} is already registered. Skipping.`); } From 1e122091a43fa14b62f86c73541211f291a46c86 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 15 Jul 2025 10:48:14 +0200 Subject: [PATCH 016/169] removed log --- packages/service-core/src/emitters/EmitterEngine.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 611496811..51a1dd0ed 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -30,7 +30,6 @@ export class EmitterEngine implements BaseEmitterEngine { bindEvent(event: event_types.EmitterEvent): void { const eventNames = this.emitter.eventNames(); - console.log(event); if (!eventNames.includes(event.name)) { logger.info(`Registering event: ${event.name}`); this.eventsMap.set(event.name, event); From 499d98b6e23c56827a01b9d93d343712e4c33ceb Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 15 Jul 2025 15:17:45 +0200 Subject: [PATCH 017/169] report storage initial --- .../src/storage/MongoReportStorage.ts | 28 +++++++++++++++++ .../implementation/MongoStorageProvider.ts | 11 +++++-- .../src/storage/implementation/db.ts | 3 ++ .../src/storage/implementation/models.ts | 9 ++++++ .../src/storage/PostgresStorageProvider.ts | 1 + .../src/storage/ReportStorageFactory.ts | 11 +++++++ .../src/storage/StorageProvider.ts | 3 ++ .../service-core/src/storage/storage-index.ts | 1 + packages/types/src/events.ts | 31 +++++++++++++++++++ 9 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/MongoReportStorage.ts create mode 100644 packages/service-core/src/storage/ReportStorageFactory.ts diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts new file mode 100644 index 000000000..abfa7a72b --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -0,0 +1,28 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { storage } from '@powersync/service-core'; +import { event_types } from '@powersync/service-types'; +import { PowerSyncMongo } from './implementation/db.js'; + +export class MongoReportStorage implements storage.ReportStorageFactory { + private readonly client: mongo.MongoClient; + public readonly db: PowerSyncMongo; + + constructor(db: PowerSyncMongo) { + this.client = db.client; + this.db = db; + } + + async reportSdkConnect(data: event_types.SdkConnectEventData): Promise { + console.log('MongoReportStorage.reportSdkConnect', data); + } + async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { + console.log('MongoReportStorage.reportSdkDisconnect', data); + } + async listCurrentConnections(data: event_types.CurrentConnectionsData): Promise { + console.log('MongoReportStorage.listCurrentConnections', data); + } + + async [Symbol.asyncDispose]() { + // No-op + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts index 75333edb1..5fac9c198 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts @@ -4,6 +4,7 @@ import { POWERSYNC_VERSION, storage } from '@powersync/service-core'; import { MongoStorageConfig } from '../../types/types.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; import { PowerSyncMongo } from './db.js'; +import { MongoReportStorage } from '../MongoReportStorage.js'; export class MongoStorageProvider implements storage.BucketStorageProvider { get type() { @@ -37,15 +38,19 @@ export class MongoStorageProvider implements storage.BucketStorageProvider { await client.connect(); const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database }); - const factory = new MongoBucketStorage(database, { + const syncStorageFactory = new MongoBucketStorage(database, { // TODO currently need the entire resolved config due to this slot_name_prefix: resolvedConfig.slot_name_prefix }); + + // TODO: CREATE REPORT STORAGE FACTORY + const reportStorageFactory = new MongoReportStorage(database); return { - storage: factory, + storage: syncStorageFactory, + reportStorage: reportStorageFactory, shutDown: async () => { shuttingDown = true; - await factory[Symbol.asyncDispose](); + await syncStorageFactory[Symbol.asyncDispose](); await client.close(); }, tearDown: () => { diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index dc5f4738e..8a56a94cb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -12,6 +12,7 @@ import { CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, + SdkConnectEventDocument, SourceTableDocument, SyncRuleDocument, WriteCheckpointDocument @@ -37,6 +38,7 @@ export class PowerSyncMongo { readonly locks: mongo.Collection; readonly bucket_state: mongo.Collection; readonly checkpoint_events: mongo.Collection; + readonly sdk_connect_events: mongo.Collection; readonly client: mongo.MongoClient; readonly db: mongo.Db; @@ -61,6 +63,7 @@ export class PowerSyncMongo { this.locks = this.db.collection('locks'); this.bucket_state = this.db.collection('bucket_state'); this.checkpoint_events = this.db.collection('checkpoint_events'); + this.sdk_connect_events = this.db.collection('sdk_connect_events'); } /** diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 181d50fff..129c7c761 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -213,3 +213,12 @@ export interface InstanceDocument { // The instance UUID _id: string; } + +export interface SdkConnectEventDocument { + version: string; + sdk: string; + user_agent: string; + client_id: string; + user_id: string; + jwt_exp: number; +} diff --git a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts index 7125ec6a8..15a66e4e8 100644 --- a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts +++ b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts @@ -29,6 +29,7 @@ export class PostgresStorageProvider implements storage.BucketStorageProvider { slot_name_prefix: options.resolvedConfig.slot_name_prefix }); return { + reportStorage: null, storage: storageFactory, shutDown: async () => storageFactory.db[Symbol.asyncDispose](), tearDown: async () => { diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts new file mode 100644 index 000000000..022cb1563 --- /dev/null +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -0,0 +1,11 @@ +import { + CurrentConnectionsData, + SdkConnectEventData, + SdkDisconnectEventData +} from '@powersync/service-types/dist/events.js'; + +export interface ReportStorageFactory extends AsyncDisposable { + reportSdkConnect(data: SdkConnectEventData): Promise; + reportSdkDisconnect(data: SdkDisconnectEventData): Promise; + listCurrentConnections(data: CurrentConnectionsData): Promise; +} diff --git a/packages/service-core/src/storage/StorageProvider.ts b/packages/service-core/src/storage/StorageProvider.ts index 4cffbb1ec..3078438bb 100644 --- a/packages/service-core/src/storage/StorageProvider.ts +++ b/packages/service-core/src/storage/StorageProvider.ts @@ -1,9 +1,12 @@ import { ServiceError } from '@powersync/lib-services-framework'; import * as util from '../util/util-index.js'; import { BucketStorageFactory } from './BucketStorageFactory.js'; +import { ReportStorageFactory } from './ReportStorageFactory.js'; export interface ActiveStorage { storage: BucketStorageFactory; + // TODO: REMOVE THE NULL ONCE POSTGRES HAS BEEN IMPLEMENTED + reportStorage: ReportStorageFactory | null; shutDown(): Promise; /** diff --git a/packages/service-core/src/storage/storage-index.ts b/packages/service-core/src/storage/storage-index.ts index 9485f85d5..5e2cb99d5 100644 --- a/packages/service-core/src/storage/storage-index.ts +++ b/packages/service-core/src/storage/storage-index.ts @@ -13,3 +13,4 @@ export * from './BucketStorageBatch.js'; export * from './SyncRulesBucketStorage.js'; export * from './PersistedSyncRulesContent.js'; export * from './ReplicationLock.js'; +export * from './ReportStorageFactory.js'; diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 65c85b951..d9106e4c5 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -2,7 +2,38 @@ export enum EmitterEngineEventNames { SDK_CONNECT_EVENT = 'sdk-connect-event', SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' } +type JwtPayload = { + sub: string; + iss?: string | undefined; + exp: number; + iat: number; +}; +export type SdkConnectEventData = { + type: EmitterEngineEventNames.SDK_DISCONNECT_EVENT; + client_id?: string; + user_id: string; + connect_at: number; + user_agent: string; + jwt_token?: JwtPayload; +}; + +export type SdkDisconnectEventData = { + type: EmitterEngineEventNames.SDK_DISCONNECT_EVENT; + client_id?: string; + user_id: string; + disconnect_at: number; + user_agent: string; + jwt_token?: JwtPayload; +}; + +export type CurrentConnectionsData = { + app_id: string; + org_id: string; + id: string; + cursor: string; + limit?: number; +}; export type EventHandlerFunc = (data: any) => Promise | void; export interface EmitterEvent { name: EmitterEngineEventNames; From 44209ff63a1bc6498c67828170d4843b7be4adfb Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 15 Jul 2025 15:34:47 +0200 Subject: [PATCH 018/169] sdk scape --- .../src/storage/MongoReportStorage.ts | 6 +++++- packages/service-core/src/storage/ReportStorageFactory.ts | 5 +++-- packages/types/src/events.ts | 5 ++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index abfa7a72b..2a0e3e24a 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -12,13 +12,17 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.db = db; } + async scrapeSdkData(data: event_types.PaginatedInstanceRequest): Promise { + console.log('MongoReportStorage.scrapeSdkData', data); + } + async reportSdkConnect(data: event_types.SdkConnectEventData): Promise { console.log('MongoReportStorage.reportSdkConnect', data); } async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { console.log('MongoReportStorage.reportSdkDisconnect', data); } - async listCurrentConnections(data: event_types.CurrentConnectionsData): Promise { + async listCurrentConnections(data: event_types.PaginatedInstanceRequest): Promise { console.log('MongoReportStorage.listCurrentConnections', data); } diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index 022cb1563..ea19074c7 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -1,5 +1,5 @@ import { - CurrentConnectionsData, + PaginatedInstanceRequest, SdkConnectEventData, SdkDisconnectEventData } from '@powersync/service-types/dist/events.js'; @@ -7,5 +7,6 @@ import { export interface ReportStorageFactory extends AsyncDisposable { reportSdkConnect(data: SdkConnectEventData): Promise; reportSdkDisconnect(data: SdkDisconnectEventData): Promise; - listCurrentConnections(data: CurrentConnectionsData): Promise; + listCurrentConnections(data: PaginatedInstanceRequest): Promise; + scrapeSdkData(data: PaginatedInstanceRequest): Promise; } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index d9106e4c5..5346008b6 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -27,11 +27,10 @@ export type SdkDisconnectEventData = { jwt_token?: JwtPayload; }; -export type CurrentConnectionsData = { +export type PaginatedInstanceRequest = { app_id: string; org_id: string; - id: string; - cursor: string; + cursor?: string; limit?: number; }; export type EventHandlerFunc = (data: any) => Promise | void; From 063e44663399e0444686593aefd1e58aaeb6212f Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 11:23:32 +0200 Subject: [PATCH 019/169] reworked the emitter engine --- .../src/emitters/EmitterEngine.ts | 43 +++++++------------ .../src/emitters/emitter-interfaces.ts | 28 +++--------- .../src/routes/endpoints/socket-route.ts | 8 ++-- .../src/routes/endpoints/sync-stream.ts | 12 +++--- packages/types/src/events.ts | 42 +++++++++--------- 5 files changed, 53 insertions(+), 80 deletions(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 51a1dd0ed..cc7395fcd 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -1,11 +1,11 @@ -import { BaseEmitterEngine } from './emitter-interfaces.js'; import EventEmitter from 'node:events'; import { logger } from '@powersync/lib-services-framework'; import { event_types } from '@powersync/service-types'; +import { BaseEmitterEngine } from './emitter-interfaces.js'; export class EmitterEngine implements BaseEmitterEngine { private emitter: EventEmitter; - eventsMap: Map = new Map(); + events: Set = new Set(); constructor() { this.emitter = new EventEmitter({ captureRejections: true }); this.emitter.on('error', (error: Error) => { @@ -13,43 +13,30 @@ export class EmitterEngine implements BaseEmitterEngine { }); } - eventNames(): event_types.EmitterEngineEventNames[] { - return this.emitter.eventNames() as event_types.EmitterEngineEventNames[]; - } - - getStoredEvent(eventName: event_types.EmitterEngineEventNames): event_types.EmitterEvent { - if (!this.eventsMap.has(eventName)) { - throw new Error(`Event ${eventName} is not registered.`); + subscribe(event: event_types.EmitterEvent): void { + if (!this.events.has(event.name)) { + this.events.add(event.name); } - return this.eventsMap.get(eventName) as event_types.EmitterEvent; + this.emitter.on(event.name, event.handler.bind(event)); } - get listEvents(): event_types.EmitterEngineEventNames[] { - return this.emitter.eventNames() as event_types.EmitterEngineEventNames[]; + get listEvents(): event_types.EmitterEngineEvents[] { + return this.emitter.eventNames() as event_types.EmitterEngineEvents[]; } - bindEvent(event: event_types.EmitterEvent): void { - const eventNames = this.emitter.eventNames(); - if (!eventNames.includes(event.name)) { - logger.info(`Registering event: ${event.name}`); - this.eventsMap.set(event.name, event); - this.emitter.on(event.name, event.handler.bind(event)); - } else { - logger.warn(`Event ${event.name} is already registered. Skipping.`); - } + countListeners(eventName: event_types.EmitterEngineEvents): number { + return this.emitter.listenerCount(eventName); } - emitEvent(eventName: event_types.EmitterEngineEventNames, data: any): void { - if (!this.emitter.eventNames().includes(eventName)) { - logger.error(`Event ${eventName} is not registered.`); - } else { - logger.info(`Emitting event: ${eventName}`, data); - this.emitter.emit(eventName, { ...data, type: eventName }); + emit(event: K, data: event_types.SubscribeEvents[K]): void { + if (!this.events.has(event) || this.countListeners(event) === 0) { + logger.warn(`${event} has no listener registered.`); } + this.emitter.emit(event, { ...data, type: event }); } shutDown(): void { + logger.info(`Shutting down EmitterEngine and removing all listeners for ${this.listEvents}.`); this.emitter.removeAllListeners(); - logger.info('Emitter engine shut down and all listeners removed.'); } } diff --git a/packages/service-core/src/emitters/emitter-interfaces.ts b/packages/service-core/src/emitters/emitter-interfaces.ts index ac49b65fe..0b75e9c5f 100644 --- a/packages/service-core/src/emitters/emitter-interfaces.ts +++ b/packages/service-core/src/emitters/emitter-interfaces.ts @@ -1,26 +1,12 @@ -// export type SdkEvent = { -// client_id?: string; -// user_id: string; -// user_agent: string; -// jwt_token?: JwtPayload; -// }; -// -// export type EventConnectData = { -// type: EmitterEngineEventNames.SDK_CONNECT_EVENT; -// connect_at: number; -// } & SdkEvent; -// export type EventDisconnectData = { -// type: EmitterEngineEventNames.SDK_DISCONNECT_EVENT; -// disconnect_at: number; -// } & SdkEvent; - import { event_types } from '@powersync/service-types'; export interface BaseEmitterEngine { - listEvents: event_types.EmitterEngineEventNames[]; - bindEvent(events: event_types.EmitterEvent): void; - eventNames(): event_types.EmitterEngineEventNames[]; - emitEvent(eventName: event_types.EmitterEngineEventNames, data: any): void; - getStoredEvent(eventName: event_types.EmitterEngineEventNames): event_types.EmitterEvent; + listEvents: event_types.EmitterEngineEvents[]; + countListeners(eventName: event_types.EmitterEngineEvents): number; + emit( + event: event_types.EmitterEngineEvents, + data: event_types.SubscribeEvents[K] + ): void; + subscribe(event: event_types.EmitterEvent): void; shutDown(): void; } diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 45df161c5..76b31a20c 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -23,11 +23,11 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => user_agent: context.user_agent }; - const sdkData = { + const sdkData: event_types.SdkUserData = { client_id: params.client_id, user_id: context.user_id!, user_agent: context.user_agent, - jwt_token: context.token_payload + jwt_exp: { exp: context.token_payload?.exp } }; const streamStart = Date.now(); @@ -94,7 +94,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => }); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); - service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_CONNECT_EVENT, { + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, { ...sdkData, connect_at: streamStart }); @@ -177,7 +177,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => close_reason: closeReason ?? 'unknown' }); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_DISCONNECT_EVENT, { + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { ...sdkData, disconnect_at: Date.now() }); diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 621ec348d..0f487a7ac 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -33,11 +33,13 @@ export const syncStreamed = routeDefinition({ user_id: payload.context.user_id }; - const sdkData = { + const sdkData: event_types.SdkUserData = { client_id: clientId, user_id: payload.context.user_id!, user_agent: userAgent as string, - jwt_token: token_payload + jwt_exp: { + exp: token_payload?.exp + } }; if (routerEngine.closed) { @@ -67,7 +69,7 @@ export const syncStreamed = routeDefinition({ const tracker = new sync.RequestTracker(metricsEngine); try { metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); - service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_CONNECT_EVENT, { + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, { ...sdkData, connect_at: streamStart }); @@ -133,7 +135,7 @@ export const syncStreamed = routeDefinition({ } controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_DISCONNECT_EVENT, { + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { ...sdkData, disconnect_at: Date.now() }); @@ -147,7 +149,7 @@ export const syncStreamed = routeDefinition({ } catch (ex) { controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emitEvent(event_types.EmitterEngineEventNames.SDK_DISCONNECT_EVENT, { + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { ...sdkData, disconnect_at: Date.now() }); diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 5346008b6..60d5034e5 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -1,31 +1,30 @@ -export enum EmitterEngineEventNames { +export enum EmitterEngineEvents { SDK_CONNECT_EVENT = 'sdk-connect-event', SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' } -type JwtPayload = { - sub: string; - iss?: string | undefined; - exp: number; - iat: number; +type JwtExp = { + exp?: number; }; -export type SdkConnectEventData = { - type: EmitterEngineEventNames.SDK_DISCONNECT_EVENT; +export type SubscribeEvents = { + [EmitterEngineEvents.SDK_CONNECT_EVENT]: SdkConnectEventData; + [EmitterEngineEvents.SDK_DISCONNECT_EVENT]: SdkDisconnectEventData; +}; + +export type SdkUserData = { client_id?: string; user_id: string; - connect_at: number; - user_agent: string; - jwt_token?: JwtPayload; + user_agent?: string; + jwt_exp: JwtExp; }; +export type SdkConnectEventData = { + connect_at: number; +} & SdkUserData; + export type SdkDisconnectEventData = { - type: EmitterEngineEventNames.SDK_DISCONNECT_EVENT; - client_id?: string; - user_id: string; disconnect_at: number; - user_agent: string; - jwt_token?: JwtPayload; -}; +} & SdkUserData; export type PaginatedInstanceRequest = { app_id: string; @@ -33,9 +32,8 @@ export type PaginatedInstanceRequest = { cursor?: string; limit?: number; }; -export type EventHandlerFunc = (data: any) => Promise | void; -export interface EmitterEvent { - name: EmitterEngineEventNames; - setController?: (controller: any) => EmitterEvent; - handler: EventHandlerFunc; +export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; +export interface EmitterEvent { + name: EmitterEngineEvents; + handler: EventHandlerFunc; } From e3e7aabd09ab92cfa3ebd639c357c7cdf843c286 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 11:39:37 +0200 Subject: [PATCH 020/169] alterations to types eventemitter --- packages/service-core/src/emitters/EmitterEngine.ts | 6 +++--- packages/types/src/events.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index cc7395fcd..0ac63085d 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -14,10 +14,10 @@ export class EmitterEngine implements BaseEmitterEngine { } subscribe(event: event_types.EmitterEvent): void { - if (!this.events.has(event.name)) { - this.events.add(event.name); + if (!this.events.has(event.event)) { + this.events.add(event.event); } - this.emitter.on(event.name, event.handler.bind(event)); + this.emitter.on(event.event, event.handler.bind(event)); } get listEvents(): event_types.EmitterEngineEvents[] { diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 60d5034e5..3c8da69d6 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -34,6 +34,6 @@ export type PaginatedInstanceRequest = { }; export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; export interface EmitterEvent { - name: EmitterEngineEvents; + event: K; handler: EventHandlerFunc; } From 5359d09f3aa1e6b12fb34cf0c60f4af9d03a4065 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 12:05:25 +0200 Subject: [PATCH 021/169] list events chnage --- packages/service-core/src/emitters/EmitterEngine.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 0ac63085d..3d1736f74 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -21,7 +21,7 @@ export class EmitterEngine implements BaseEmitterEngine { } get listEvents(): event_types.EmitterEngineEvents[] { - return this.emitter.eventNames() as event_types.EmitterEngineEvents[]; + return Array.from(this.events.values()); } countListeners(eventName: event_types.EmitterEngineEvents): number { @@ -36,7 +36,7 @@ export class EmitterEngine implements BaseEmitterEngine { } shutDown(): void { - logger.info(`Shutting down EmitterEngine and removing all listeners for ${this.listEvents}.`); + logger.info(`Shutting down EmitterEngine and removing all listeners for ${this.listEvents.join(', ')}.`); this.emitter.removeAllListeners(); } } From ce5092b1e6f933e5da3f51c0986f66ebbb25cede Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 13:48:59 +0200 Subject: [PATCH 022/169] changed document type mongo --- .../migrations/1752661449910-sdk-reporting.ts | 63 +++++++++++++++++++ .../src/storage/MongoReportStorage.ts | 7 ++- .../src/storage/implementation/db.ts | 20 +++++- .../src/storage/implementation/models.ts | 10 +-- .../src/routes/endpoints/socket-route.ts | 2 + .../src/routes/endpoints/sync-stream.ts | 3 +- .../src/storage/ReportStorageFactory.ts | 10 +-- packages/types/src/events.ts | 19 +++++- 8 files changed, 110 insertions(+), 24 deletions(-) create mode 100644 modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts diff --git a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts new file mode 100644 index 000000000..b9d516a9d --- /dev/null +++ b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts @@ -0,0 +1,63 @@ +import { migrations } from '@powersync/service-core'; +import * as storage from '../../../storage/storage-index.js'; +import { MongoStorageConfig } from '../../../types/types.js'; + +export const up: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig); + + try { + await db.createSdkReportingCollection(); + + await db.sdk_report_events.createIndex( + { + connect_at: 1, + jwt_exp: 1, + disconnect_at: 1 + }, + { name: 'connect_at' } + ); + + await db.sdk_report_events.createIndex( + { + user_id: 1, + sdk: 1, + version: 1 + }, + { name: 'user_id' } + ); + await db.sdk_report_events.createIndex( + { + client_id: 1 + }, + { name: 'client_id' } + ); + } finally { + await db.client.close(); + } +}; + +export const down: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + + const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig); + + try { + if (await db.write_checkpoints.indexExists('connect_at')) { + await db.write_checkpoints.dropIndex('connect_at'); + } + if (await db.custom_write_checkpoints.indexExists('user_id')) { + await db.custom_write_checkpoints.dropIndex('user_id'); + } + if (await db.custom_write_checkpoints.indexExists('client_id')) { + await db.custom_write_checkpoints.dropIndex('client_id'); + } + await db.db.dropCollection('sdk_report_events'); + } finally { + await db.client.close(); + } +}; diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 2a0e3e24a..41ee85018 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -2,6 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; +import { SdkConnectDocument } from './implementation/models.js'; export class MongoReportStorage implements storage.ReportStorageFactory { private readonly client: mongo.MongoClient; @@ -16,10 +17,10 @@ export class MongoReportStorage implements storage.ReportStorageFactory { console.log('MongoReportStorage.scrapeSdkData', data); } - async reportSdkConnect(data: event_types.SdkConnectEventData): Promise { - console.log('MongoReportStorage.reportSdkConnect', data); + async reportSdkConnect(data: SdkConnectDocument): Promise { + await this.db.sdk_report_events.insertOne(data); } - async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { + async reportSdkDisconnect(data: SdkConnectDocument): Promise { console.log('MongoReportStorage.reportSdkDisconnect', data); } async listCurrentConnections(data: event_types.PaginatedInstanceRequest): Promise { diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 8a56a94cb..a95e9fb9b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -12,7 +12,7 @@ import { CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, - SdkConnectEventDocument, + SdkConnectDocument, SourceTableDocument, SyncRuleDocument, WriteCheckpointDocument @@ -38,7 +38,7 @@ export class PowerSyncMongo { readonly locks: mongo.Collection; readonly bucket_state: mongo.Collection; readonly checkpoint_events: mongo.Collection; - readonly sdk_connect_events: mongo.Collection; + readonly sdk_report_events: mongo.Collection; readonly client: mongo.MongoClient; readonly db: mongo.Db; @@ -63,7 +63,7 @@ export class PowerSyncMongo { this.locks = this.db.collection('locks'); this.bucket_state = this.db.collection('bucket_state'); this.checkpoint_events = this.db.collection('checkpoint_events'); - this.sdk_connect_events = this.db.collection('sdk_connect_events'); + this.sdk_report_events = this.db.collection('sdk_report_events'); } /** @@ -130,6 +130,20 @@ export class PowerSyncMongo { max: 50 // max number of documents }); } + + /** + * Only use in migrations and tests. + */ + async createSdkReportingCollection() { + const existingCollections = await this.db + .listCollections({ name: 'sdk_report_events' }, { nameOnly: false }) + .toArray(); + const collection = existingCollections[0]; + if (collection != null) { + return; + } + await this.db.createCollection('sdk_report_events'); + } } export function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 129c7c761..0a87b9768 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -1,6 +1,7 @@ import { InternalOpId, storage } from '@powersync/service-core'; import { SqliteJsonValue } from '@powersync/service-sync-rules'; import * as bson from 'bson'; +import { event_types } from '@powersync/service-types'; /** * Replica id uniquely identifying a row on the source database. @@ -214,11 +215,4 @@ export interface InstanceDocument { _id: string; } -export interface SdkConnectEventDocument { - version: string; - sdk: string; - user_agent: string; - client_id: string; - user_id: string; - jwt_exp: number; -} +export interface SdkConnectDocument extends event_types.SdkConnectDocument {} diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 76b31a20c..62d0feda4 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -1,5 +1,6 @@ import { ErrorCode, errors, schema } from '@powersync/lib-services-framework'; import { RequestParameters } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; import { serialize } from 'bson'; import * as sync from '../../sync/sync-index.js'; @@ -24,6 +25,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => }; const sdkData: event_types.SdkUserData = { + _id: new bson.ObjectId(), client_id: params.client_id, user_id: context.user_id!, user_agent: context.user_agent, diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 0f487a7ac..2964d55bb 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -1,7 +1,7 @@ import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework'; import { RequestParameters } from '@powersync/service-sync-rules'; import { Readable } from 'stream'; - +import * as bson from 'bson'; import * as sync from '../../sync/sync-index.js'; import * as util from '../../util/util-index.js'; @@ -34,6 +34,7 @@ export const syncStreamed = routeDefinition({ }; const sdkData: event_types.SdkUserData = { + _id: new bson.ObjectId(), client_id: clientId, user_id: payload.context.user_id!, user_agent: userAgent as string, diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index ea19074c7..4b262ecb5 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -1,12 +1,8 @@ -import { - PaginatedInstanceRequest, - SdkConnectEventData, - SdkDisconnectEventData -} from '@powersync/service-types/dist/events.js'; +import { PaginatedInstanceRequest, SdkConnectDocument } from '@powersync/service-types/dist/events.js'; export interface ReportStorageFactory extends AsyncDisposable { - reportSdkConnect(data: SdkConnectEventData): Promise; - reportSdkDisconnect(data: SdkDisconnectEventData): Promise; + reportSdkConnect(data: SdkConnectDocument): Promise; + reportSdkDisconnect(data: SdkConnectDocument): Promise; listCurrentConnections(data: PaginatedInstanceRequest): Promise; scrapeSdkData(data: PaginatedInstanceRequest): Promise; } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 3c8da69d6..14d01f76c 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -1,3 +1,5 @@ +import * as bson from 'bson'; + export enum EmitterEngineEvents { SDK_CONNECT_EVENT = 'sdk-connect-event', SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' @@ -12,6 +14,7 @@ export type SubscribeEvents = { }; export type SdkUserData = { + _id: bson.ObjectId; client_id?: string; user_id: string; user_agent?: string; @@ -19,13 +22,25 @@ export type SdkUserData = { }; export type SdkConnectEventData = { - connect_at: number; + connect_at: Date; } & SdkUserData; export type SdkDisconnectEventData = { - disconnect_at: number; + disconnect_at: Date; } & SdkUserData; +export type SdkConnectDocument = { + _id: bson.ObjectId; + sdk: string; + version: string; + user_agent: string; + client_id: string; + user_id: string; + jwt_exp?: Date; + connect_at: Date; + disconnect_at?: Date; +}; + export type PaginatedInstanceRequest = { app_id: string; org_id: string; From 5ff2da5f4641abd0e7c31ef744a3718ee707657c Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 13:53:03 +0200 Subject: [PATCH 023/169] changed document type mongo --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 4 +++- packages/types/src/events.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 41ee85018..b953eb618 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -18,10 +18,12 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async reportSdkConnect(data: SdkConnectDocument): Promise { + console.log(data); await this.db.sdk_report_events.insertOne(data); } async reportSdkDisconnect(data: SdkConnectDocument): Promise { - console.log('MongoReportStorage.reportSdkDisconnect', data); + console.log(data); + await this.db.sdk_report_events.findOneAndUpdate({ _id: data._id }, data, { upsert: true }); } async listCurrentConnections(data: event_types.PaginatedInstanceRequest): Promise { console.log('MongoReportStorage.listCurrentConnections', data); diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 14d01f76c..554672094 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -37,7 +37,7 @@ export type SdkConnectDocument = { client_id: string; user_id: string; jwt_exp?: Date; - connect_at: Date; + connect_at?: Date; disconnect_at?: Date; }; From 0657ffe85e1f8dcb9012c03e0f4e85a2f9308d57 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 13:56:43 +0200 Subject: [PATCH 024/169] changed document type mongo --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index b953eb618..a358592db 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -22,8 +22,8 @@ export class MongoReportStorage implements storage.ReportStorageFactory { await this.db.sdk_report_events.insertOne(data); } async reportSdkDisconnect(data: SdkConnectDocument): Promise { - console.log(data); - await this.db.sdk_report_events.findOneAndUpdate({ _id: data._id }, data, { upsert: true }); + const { _id, ...rest } = data; + await this.db.sdk_report_events.findOneAndUpdate({ _id }, rest, { upsert: true }); } async listCurrentConnections(data: event_types.PaginatedInstanceRequest): Promise { console.log('MongoReportStorage.listCurrentConnections', data); From 388ab2fed5c381c2178dcfab2647af97ca66a50f Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 13:56:54 +0200 Subject: [PATCH 025/169] changed document type mongo --- modules/module-mongodb-storage/src/storage/MongoReportStorage.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index a358592db..2c9be11ad 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -18,7 +18,6 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async reportSdkConnect(data: SdkConnectDocument): Promise { - console.log(data); await this.db.sdk_report_events.insertOne(data); } async reportSdkDisconnect(data: SdkConnectDocument): Promise { From af06f645d95e3f7047d3f14ba1fd72589fc3d0b2 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 14:00:45 +0200 Subject: [PATCH 026/169] chnaged date imps events --- packages/service-core/src/routes/endpoints/socket-route.ts | 4 ++-- packages/service-core/src/routes/endpoints/sync-stream.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 62d0feda4..47655c943 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -98,7 +98,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, { ...sdkData, - connect_at: streamStart + connect_at: new Date(streamStart) }); const tracker = new sync.RequestTracker(metricsEngine); try { @@ -181,7 +181,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { ...sdkData, - disconnect_at: Date.now() + disconnect_at: new Date() }); } } diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 2964d55bb..d2e24280d 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -72,7 +72,7 @@ export const syncStreamed = routeDefinition({ metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, { ...sdkData, - connect_at: streamStart + connect_at: new Date(streamStart) }); const stream = Readable.from( sync.transformToBytesTracked( @@ -138,7 +138,7 @@ export const syncStreamed = routeDefinition({ metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { ...sdkData, - disconnect_at: Date.now() + disconnect_at: new Date() }); logger.info(`Sync stream complete`, { ...tracker.getLogMeta(), @@ -152,7 +152,7 @@ export const syncStreamed = routeDefinition({ metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { ...sdkData, - disconnect_at: Date.now() + disconnect_at: new Date() }); } } From 50eb1984aa0a063643e9036265146f45b602d7e8 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 14:19:47 +0200 Subject: [PATCH 027/169] chnaged jwt exp --- packages/service-core/src/routes/endpoints/socket-route.ts | 2 +- packages/service-core/src/routes/endpoints/sync-stream.ts | 4 +--- packages/types/src/events.ts | 5 +---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 47655c943..ea810cc6e 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -29,7 +29,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => client_id: params.client_id, user_id: context.user_id!, user_agent: context.user_agent, - jwt_exp: { exp: context.token_payload?.exp } + jwt_exp: context.token_payload?.exp ? new Date(context.token_payload.exp) : undefined }; const streamStart = Date.now(); diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index d2e24280d..3d9425abf 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -38,9 +38,7 @@ export const syncStreamed = routeDefinition({ client_id: clientId, user_id: payload.context.user_id!, user_agent: userAgent as string, - jwt_exp: { - exp: token_payload?.exp - } + jwt_exp: token_payload?.exp ? new Date(token_payload?.exp) : undefined }; if (routerEngine.closed) { diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 554672094..ab776b3e0 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -4,9 +4,6 @@ export enum EmitterEngineEvents { SDK_CONNECT_EVENT = 'sdk-connect-event', SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' } -type JwtExp = { - exp?: number; -}; export type SubscribeEvents = { [EmitterEngineEvents.SDK_CONNECT_EVENT]: SdkConnectEventData; @@ -18,7 +15,7 @@ export type SdkUserData = { client_id?: string; user_id: string; user_agent?: string; - jwt_exp: JwtExp; + jwt_exp?: Date; }; export type SdkConnectEventData = { From ec9ce6f6963c800f3ada4a1fee58aca021df250e Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 14:52:13 +0200 Subject: [PATCH 028/169] ffs --- .../src/storage/MongoReportStorage.ts | 6 ++++-- packages/types/src/events.ts | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 2c9be11ad..96d269ec9 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -18,11 +18,13 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async reportSdkConnect(data: SdkConnectDocument): Promise { - await this.db.sdk_report_events.insertOne(data); + const res = await this.db.sdk_report_events.insertOne(data); + console.log(res); } async reportSdkDisconnect(data: SdkConnectDocument): Promise { const { _id, ...rest } = data; - await this.db.sdk_report_events.findOneAndUpdate({ _id }, rest, { upsert: true }); + const res = await this.db.sdk_report_events.findOneAndUpdate({ _id }, rest, { upsert: true }); + console.log(res); } async listCurrentConnections(data: event_types.PaginatedInstanceRequest): Promise { console.log('MongoReportStorage.listCurrentConnections', data); diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index ab776b3e0..fe0609a76 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -26,6 +26,7 @@ export type SdkDisconnectEventData = { disconnect_at: Date; } & SdkUserData; +// Mongodb document type for SDK connect and disconnect events export type SdkConnectDocument = { _id: bson.ObjectId; sdk: string; From 8a017f844d80da328b8147de5e4d301cd5501b5d Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 15:21:28 +0200 Subject: [PATCH 029/169] dev packages are desynced --- libs/lib-services/src/system/LifeCycledSystem.ts | 2 +- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 1 + .../src/storage/implementation/MongoStorageProvider.ts | 2 +- .../src/storage/PostgresStorageProvider.ts | 1 + packages/service-core/src/storage/ReportStorageFactory.ts | 1 + packages/service-core/src/storage/StorageProvider.ts | 2 +- packages/types/src/events.ts | 1 - 7 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libs/lib-services/src/system/LifeCycledSystem.ts b/libs/lib-services/src/system/LifeCycledSystem.ts index a179e0d3f..9d5c1e517 100644 --- a/libs/lib-services/src/system/LifeCycledSystem.ts +++ b/libs/lib-services/src/system/LifeCycledSystem.ts @@ -9,7 +9,7 @@ import { ServiceError } from '@powersync/service-errors'; import { container } from '../container.js'; import { logger } from '../logger/Logger.js'; - +// TODO: REMOVE THIS COMMENT export type LifecycleCallback = (singleton: T) => Promise | void; export type PartialLifecycle = { diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 96d269ec9..c6eccb173 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -4,6 +4,7 @@ import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; import { SdkConnectDocument } from './implementation/models.js'; +// import { SdkConnectDocument } from './implementation/models.js'; export class MongoReportStorage implements storage.ReportStorageFactory { private readonly client: mongo.MongoClient; public readonly db: PowerSyncMongo; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts index 5fac9c198..42e211303 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts @@ -43,7 +43,7 @@ export class MongoStorageProvider implements storage.BucketStorageProvider { slot_name_prefix: resolvedConfig.slot_name_prefix }); - // TODO: CREATE REPORT STORAGE FACTORY + // Storage factory for reports const reportStorageFactory = new MongoReportStorage(database); return { storage: syncStorageFactory, diff --git a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts index 15a66e4e8..12e25c184 100644 --- a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts +++ b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts @@ -29,6 +29,7 @@ export class PostgresStorageProvider implements storage.BucketStorageProvider { slot_name_prefix: options.resolvedConfig.slot_name_prefix }); return { + // TODO: IMPLEMENT REPORT STORAGE reportStorage: null, storage: storageFactory, shutDown: async () => storageFactory.db[Symbol.asyncDispose](), diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index 4b262ecb5..1583bd258 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -1,5 +1,6 @@ import { PaginatedInstanceRequest, SdkConnectDocument } from '@powersync/service-types/dist/events.js'; +// Interface for the ReportStorageFactory export interface ReportStorageFactory extends AsyncDisposable { reportSdkConnect(data: SdkConnectDocument): Promise; reportSdkDisconnect(data: SdkConnectDocument): Promise; diff --git a/packages/service-core/src/storage/StorageProvider.ts b/packages/service-core/src/storage/StorageProvider.ts index 3078438bb..f813502fa 100644 --- a/packages/service-core/src/storage/StorageProvider.ts +++ b/packages/service-core/src/storage/StorageProvider.ts @@ -5,7 +5,7 @@ import { ReportStorageFactory } from './ReportStorageFactory.js'; export interface ActiveStorage { storage: BucketStorageFactory; - // TODO: REMOVE THE NULL ONCE POSTGRES HAS BEEN IMPLEMENTED + // TODO: REMOVE THE NULL ONCE POSTGRES HAS BEEN IMPLEMENTED THIS IS JUST SO I CAN TEST MONGO reportStorage: ReportStorageFactory | null; shutDown(): Promise; diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index fe0609a76..ab776b3e0 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -26,7 +26,6 @@ export type SdkDisconnectEventData = { disconnect_at: Date; } & SdkUserData; -// Mongodb document type for SDK connect and disconnect events export type SdkConnectDocument = { _id: bson.ObjectId; sdk: string; From 63b6f67312ee8f64e8cd64d3c49126ef1aad26cc Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 15:31:42 +0200 Subject: [PATCH 030/169] test disconnect event --- .../src/storage/MongoReportStorage.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index c6eccb173..9120d45f5 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -19,13 +19,16 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async reportSdkConnect(data: SdkConnectDocument): Promise { - const res = await this.db.sdk_report_events.insertOne(data); - console.log(res); + await this.db.sdk_report_events.insertOne(data); } async reportSdkDisconnect(data: SdkConnectDocument): Promise { const { _id, ...rest } = data; - const res = await this.db.sdk_report_events.findOneAndUpdate({ _id }, rest, { upsert: true }); - console.log(res); + console.log(data); + try { + await this.db.sdk_report_events.findOneAndUpdate({ _id }, rest, { upsert: true }); + } catch (error) { + console.log(error); + } } async listCurrentConnections(data: event_types.PaginatedInstanceRequest): Promise { console.log('MongoReportStorage.listCurrentConnections', data); From d5429c78ea5c98706d9497f13331c441b006ad15 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 15:43:16 +0200 Subject: [PATCH 031/169] disconnect event --- .../src/storage/MongoReportStorage.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 9120d45f5..c1cfaed8a 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -23,12 +23,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async reportSdkDisconnect(data: SdkConnectDocument): Promise { const { _id, ...rest } = data; - console.log(data); - try { - await this.db.sdk_report_events.findOneAndUpdate({ _id }, rest, { upsert: true }); - } catch (error) { - console.log(error); - } + await this.db.sdk_report_events.findOneAndUpdate({ _id }, { $set: rest }, { upsert: true }); } async listCurrentConnections(data: event_types.PaginatedInstanceRequest): Promise { console.log('MongoReportStorage.listCurrentConnections', data); From 83d4280d37baaeaa9794741470bce309bb45b159 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 16 Jul 2025 16:38:38 +0200 Subject: [PATCH 032/169] removed accidental key in event --- packages/service-core/src/emitters/EmitterEngine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 3d1736f74..c7e21bf25 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -32,7 +32,7 @@ export class EmitterEngine implements BaseEmitterEngine { if (!this.events.has(event) || this.countListeners(event) === 0) { logger.warn(`${event} has no listener registered.`); } - this.emitter.emit(event, { ...data, type: event }); + this.emitter.emit(event, data); } shutDown(): void { From 8f74f2714caa4204d91de753e8992b704b6a878e Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 09:58:51 +0200 Subject: [PATCH 033/169] changes to update logic sdk --- .../src/storage/MongoReportStorage.ts | 22 +++++++++++++++---- .../src/storage/implementation/db.ts | 1 + .../src/routes/endpoints/socket-route.ts | 2 -- .../src/routes/endpoints/sync-stream.ts | 2 -- packages/types/src/events.ts | 4 ---- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index c1cfaed8a..186250492 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -4,7 +4,6 @@ import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; import { SdkConnectDocument } from './implementation/models.js'; -// import { SdkConnectDocument } from './implementation/models.js'; export class MongoReportStorage implements storage.ReportStorageFactory { private readonly client: mongo.MongoClient; public readonly db: PowerSyncMongo; @@ -19,11 +18,26 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async reportSdkConnect(data: SdkConnectDocument): Promise { - await this.db.sdk_report_events.insertOne(data); + await this.db.sdk_report_events.findOneAndUpdate( + { user_id: data.user_id, client_id: data.client_id }, + { + $set: data + }, + { + upsert: true + } + ); } async reportSdkDisconnect(data: SdkConnectDocument): Promise { - const { _id, ...rest } = data; - await this.db.sdk_report_events.findOneAndUpdate({ _id }, { $set: rest }, { upsert: true }); + await this.db.sdk_report_events.findOneAndUpdate( + { user_id: data.user_id, client_id: data.client_id }, + { + $set: { + disconnect_at: data.disconnect_at + } + }, + { upsert: true } + ); } async listCurrentConnections(data: event_types.PaginatedInstanceRequest): Promise { console.log('MongoReportStorage.listCurrentConnections', data); diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index a95e9fb9b..ae2c1fc91 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -81,6 +81,7 @@ export class PowerSyncMongo { await this.locks.deleteMany({}); await this.bucket_state.deleteMany({}); await this.custom_write_checkpoints.deleteMany({}); + await this.sdk_report_events.deleteMany({}); } /** diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index ea810cc6e..88191ec9b 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -1,6 +1,5 @@ import { ErrorCode, errors, schema } from '@powersync/lib-services-framework'; import { RequestParameters } from '@powersync/service-sync-rules'; -import * as bson from 'bson'; import { serialize } from 'bson'; import * as sync from '../../sync/sync-index.js'; @@ -25,7 +24,6 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => }; const sdkData: event_types.SdkUserData = { - _id: new bson.ObjectId(), client_id: params.client_id, user_id: context.user_id!, user_agent: context.user_agent, diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 3d9425abf..df2f72e65 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -1,7 +1,6 @@ import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework'; import { RequestParameters } from '@powersync/service-sync-rules'; import { Readable } from 'stream'; -import * as bson from 'bson'; import * as sync from '../../sync/sync-index.js'; import * as util from '../../util/util-index.js'; @@ -34,7 +33,6 @@ export const syncStreamed = routeDefinition({ }; const sdkData: event_types.SdkUserData = { - _id: new bson.ObjectId(), client_id: clientId, user_id: payload.context.user_id!, user_agent: userAgent as string, diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index ab776b3e0..82583b93d 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -1,5 +1,3 @@ -import * as bson from 'bson'; - export enum EmitterEngineEvents { SDK_CONNECT_EVENT = 'sdk-connect-event', SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' @@ -11,7 +9,6 @@ export type SubscribeEvents = { }; export type SdkUserData = { - _id: bson.ObjectId; client_id?: string; user_id: string; user_agent?: string; @@ -27,7 +24,6 @@ export type SdkDisconnectEventData = { } & SdkUserData; export type SdkConnectDocument = { - _id: bson.ObjectId; sdk: string; version: string; user_agent: string; From f1d187be2af3d509b420c5697e6e481dc3c834b7 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 10:12:53 +0200 Subject: [PATCH 034/169] chnageste bump --- .changeset/heavy-pianos-grin.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/heavy-pianos-grin.md diff --git a/.changeset/heavy-pianos-grin.md b/.changeset/heavy-pianos-grin.md new file mode 100644 index 000000000..897f7c639 --- /dev/null +++ b/.changeset/heavy-pianos-grin.md @@ -0,0 +1,7 @@ +--- +'@powersync/service-module-mongodb-storage': patch +'@powersync/service-core': patch +'@powersync/service-types': patch +--- + +sdk reporting From 5280565e9ec021b187bcb4af81bd3a2c76fe60a3 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 10:30:44 +0200 Subject: [PATCH 035/169] dev packages issue --- libs/lib-services/src/system/LifeCycledSystem.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/lib-services/src/system/LifeCycledSystem.ts b/libs/lib-services/src/system/LifeCycledSystem.ts index 9d5c1e517..a179e0d3f 100644 --- a/libs/lib-services/src/system/LifeCycledSystem.ts +++ b/libs/lib-services/src/system/LifeCycledSystem.ts @@ -9,7 +9,7 @@ import { ServiceError } from '@powersync/service-errors'; import { container } from '../container.js'; import { logger } from '../logger/Logger.js'; -// TODO: REMOVE THIS COMMENT + export type LifecycleCallback = (singleton: T) => Promise | void; export type PartialLifecycle = { From 8cbbd35e6806f4c09f354730110cecb15ee4b9b9 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 10:33:59 +0200 Subject: [PATCH 036/169] dev packages --- libs/lib-services/src/container.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/lib-services/src/container.ts b/libs/lib-services/src/container.ts index f591f24d9..f15a2a19a 100644 --- a/libs/lib-services/src/container.ts +++ b/libs/lib-services/src/container.ts @@ -4,10 +4,10 @@ import { ErrorReporter } from './alerts/definitions.js'; import { NoOpReporter } from './alerts/no-op-reporter.js'; import { MigrationManager } from './migrations/MigrationManager.js'; import { - ProbeModule, - TerminationHandler, createInMemoryProbe, - createTerminationHandler + createTerminationHandler, + ProbeModule, + TerminationHandler } from './signals/signals-index.js'; export enum ContainerImplementation { @@ -47,7 +47,7 @@ export type Newable = new (...args: never[]) => T; * Identifier used to get and register implementations */ export type ServiceIdentifier = string | symbol | Newable | Abstract | ContainerImplementation; - +// TODO: REMOVE THIS COMMENT, DEV PACKAGES const DEFAULT_GENERATORS: ContainerImplementationDefaultGenerators = { [ContainerImplementation.REPORTER]: () => NoOpReporter, [ContainerImplementation.PROBES]: () => createInMemoryProbe(), From a700ec9b20a571517edb0f7a20e2be51e58fc523 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 10:39:47 +0200 Subject: [PATCH 037/169] chnages to mongo storage and package bumps --- .changeset/smart-mugs-share.md | 13 +++++++++++++ libs/lib-services/src/container.ts | 1 - 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 .changeset/smart-mugs-share.md diff --git a/.changeset/smart-mugs-share.md b/.changeset/smart-mugs-share.md new file mode 100644 index 000000000..f8cfda8ea --- /dev/null +++ b/.changeset/smart-mugs-share.md @@ -0,0 +1,13 @@ +--- +'@powersync/service-module-postgres-storage': patch +'@powersync/service-module-mongodb-storage': patch +'@powersync/service-module-postgres': patch +'@powersync/service-module-mongodb': patch +'@powersync/service-core': patch +'@powersync/service-module-mysql': patch +'@powersync/service-module-core': patch +'@powersync/lib-services-framework': patch +'@powersync/service-types': patch +--- + +Reporting mongo storage added to storage engine. diff --git a/libs/lib-services/src/container.ts b/libs/lib-services/src/container.ts index f15a2a19a..c7c4102cb 100644 --- a/libs/lib-services/src/container.ts +++ b/libs/lib-services/src/container.ts @@ -47,7 +47,6 @@ export type Newable = new (...args: never[]) => T; * Identifier used to get and register implementations */ export type ServiceIdentifier = string | symbol | Newable | Abstract | ContainerImplementation; -// TODO: REMOVE THIS COMMENT, DEV PACKAGES const DEFAULT_GENERATORS: ContainerImplementationDefaultGenerators = { [ContainerImplementation.REPORTER]: () => NoOpReporter, [ContainerImplementation.PROBES]: () => createInMemoryProbe(), From 72398c2c2c46f2c7175dec5061d1e145e74ae382 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 10:49:02 +0200 Subject: [PATCH 038/169] unset disconnect on connect --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 186250492..f58fbda5c 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -21,7 +21,10 @@ export class MongoReportStorage implements storage.ReportStorageFactory { await this.db.sdk_report_events.findOneAndUpdate( { user_id: data.user_id, client_id: data.client_id }, { - $set: data + $set: data, + $unset: { + disconnect_at: '' + } }, { upsert: true From 842e62128d249c479fa25a28e19731acee9fb743 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 11:14:48 +0200 Subject: [PATCH 039/169] change document sdk --- .../src/storage/MongoReportStorage.ts | 10 +++++++--- .../service-core/src/storage/ReportStorageFactory.ts | 10 +++++++--- packages/types/src/events.ts | 7 +++---- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index f58fbda5c..35615d600 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -3,6 +3,7 @@ import { storage } from '@powersync/service-core'; import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; import { SdkConnectDocument } from './implementation/models.js'; +import { ListCurrentConnectionsResponse } from '@powersync/service-types/dist/events.js'; export class MongoReportStorage implements storage.ReportStorageFactory { private readonly client: mongo.MongoClient; @@ -13,7 +14,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.db = db; } - async scrapeSdkData(data: event_types.PaginatedInstanceRequest): Promise { + async scrapeSdkData(data: event_types.InstanceRequest): Promise { console.log('MongoReportStorage.scrapeSdkData', data); } @@ -37,13 +38,16 @@ export class MongoReportStorage implements storage.ReportStorageFactory { { $set: { disconnect_at: data.disconnect_at + }, + $unset: { + jwt_exp: '' } }, { upsert: true } ); } - async listCurrentConnections(data: event_types.PaginatedInstanceRequest): Promise { - console.log('MongoReportStorage.listCurrentConnections', data); + async listCurrentConnections(data: event_types.InstanceRequest): Promise { + return this.db.sdk_report_events.aggregate([{ $match: {} }]); } async [Symbol.asyncDispose]() { diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index 1583bd258..5d4ea83eb 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -1,9 +1,13 @@ -import { PaginatedInstanceRequest, SdkConnectDocument } from '@powersync/service-types/dist/events.js'; +import { + InstanceRequest, + ListCurrentConnectionsResponse, + SdkConnectDocument +} from '@powersync/service-types/dist/events.js'; // Interface for the ReportStorageFactory export interface ReportStorageFactory extends AsyncDisposable { reportSdkConnect(data: SdkConnectDocument): Promise; reportSdkDisconnect(data: SdkConnectDocument): Promise; - listCurrentConnections(data: PaginatedInstanceRequest): Promise; - scrapeSdkData(data: PaginatedInstanceRequest): Promise; + listCurrentConnections(data: InstanceRequest): Promise; + scrapeSdkData(data: InstanceRequest): Promise; } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 82583b93d..f6582e3d0 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -25,7 +25,6 @@ export type SdkDisconnectEventData = { export type SdkConnectDocument = { sdk: string; - version: string; user_agent: string; client_id: string; user_id: string; @@ -34,11 +33,11 @@ export type SdkConnectDocument = { disconnect_at?: Date; }; -export type PaginatedInstanceRequest = { +export type ListCurrentConnectionsResponse = {}; + +export type InstanceRequest = { app_id: string; org_id: string; - cursor?: string; - limit?: number; }; export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; export interface EmitterEvent { From 12ac8321eb666ef10ad605f0b2471ab7625ddb07 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 11:35:09 +0200 Subject: [PATCH 040/169] deisconnect chnages --- .../src/storage/MongoReportStorage.ts | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 35615d600..439ce196f 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -22,10 +22,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { await this.db.sdk_report_events.findOneAndUpdate( { user_id: data.user_id, client_id: data.client_id }, { - $set: data, - $unset: { - disconnect_at: '' - } + $set: data }, { upsert: true @@ -33,21 +30,17 @@ export class MongoReportStorage implements storage.ReportStorageFactory { ); } async reportSdkDisconnect(data: SdkConnectDocument): Promise { - await this.db.sdk_report_events.findOneAndUpdate( - { user_id: data.user_id, client_id: data.client_id }, - { - $set: { - disconnect_at: data.disconnect_at - }, - $unset: { - jwt_exp: '' - } - }, - { upsert: true } - ); + await this.db.sdk_report_events.findOneAndDelete({ user_id: data.user_id, client_id: data.client_id }); } async listCurrentConnections(data: event_types.InstanceRequest): Promise { - return this.db.sdk_report_events.aggregate([{ $match: {} }]); + return this.db.sdk_report_events.aggregate([ + { + $group: { + user_id: '$user_id', + client_id: '$client_id' + } + } + ]); } async [Symbol.asyncDispose]() { From 6dfe510586a8776508ad95a5432f8a5736ff6a90 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 12:05:32 +0200 Subject: [PATCH 041/169] connection query --- .../src/storage/MongoReportStorage.ts | 31 ++++++++++++++----- packages/types/src/events.ts | 6 ++-- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 439ce196f..28b082bb4 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -33,14 +33,31 @@ export class MongoReportStorage implements storage.ReportStorageFactory { await this.db.sdk_report_events.findOneAndDelete({ user_id: data.user_id, client_id: data.client_id }); } async listCurrentConnections(data: event_types.InstanceRequest): Promise { - return this.db.sdk_report_events.aggregate([ - { - $group: { - user_id: '$user_id', - client_id: '$client_id' + const result = await this.db.sdk_report_events + .aggregate([ + { + $group: { + _id: null, + user_ids: { $addToSet: '$user_id' }, + client_ids: { $addToSet: '$client_id' }, + sdks: { $addToSet: '$sdk' } + } + }, + { + $project: { + _id: 0, + user_count: { $size: '$user_ids' }, + client_id_count: { $size: '$client_ids' }, + sdk: '$sdks' + } } - } - ]); + ]) + .toArray(); + console.log(result); + return { + ...data, + ...result[0] + }; } async [Symbol.asyncDispose]() { diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index f6582e3d0..edbea68e5 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -33,12 +33,14 @@ export type SdkConnectDocument = { disconnect_at?: Date; }; -export type ListCurrentConnectionsResponse = {}; - export type InstanceRequest = { app_id: string; org_id: string; + user_count: number; + client_id_count: number; + sdk: string[]; }; +export type ListCurrentConnectionsResponse = {} & InstanceRequest; export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; export interface EmitterEvent { event: K; From f3c356ad09832dfc3ea7cc295d31320f55bbcbf9 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 12:39:48 +0200 Subject: [PATCH 042/169] chnaged query for sdk data --- .../src/storage/MongoReportStorage.ts | 42 +++++++++++++++---- .../src/storage/ReportStorageFactory.ts | 2 +- packages/types/src/events.ts | 17 +++++--- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 28b082bb4..c31dc9f5f 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -3,7 +3,7 @@ import { storage } from '@powersync/service-core'; import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; import { SdkConnectDocument } from './implementation/models.js'; -import { ListCurrentConnectionsResponse } from '@powersync/service-types/dist/events.js'; +import { ListCurrentConnections, ListCurrentConnectionsResponse } from '@powersync/service-types/dist/events.js'; export class MongoReportStorage implements storage.ReportStorageFactory { private readonly client: mongo.MongoClient; @@ -14,8 +14,32 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.db = db; } - async scrapeSdkData(data: event_types.InstanceRequest): Promise { - console.log('MongoReportStorage.scrapeSdkData', data); + async scrapeSdkData(data: event_types.InstanceRequest): Promise { + const result = await this.db.sdk_report_events + .aggregate([ + { + $group: { + _id: null, + user_ids: { $addToSet: '$user_id' }, + client_ids: { $addToSet: '$client_id' }, + sdks: { $addToSet: '$sdk' } + } + }, + { + $project: { + _id: 0, + users: '$user_ids', + clients: '$client_ids', + sdks: '$sdks' + } + } + ]) + .toArray(); + return { + app_id: data.app_id, + org_id: data.org_id, + ...result[0] + }; } async reportSdkConnect(data: SdkConnectDocument): Promise { @@ -34,7 +58,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async listCurrentConnections(data: event_types.InstanceRequest): Promise { const result = await this.db.sdk_report_events - .aggregate([ + .aggregate([ { $group: { _id: null, @@ -46,16 +70,16 @@ export class MongoReportStorage implements storage.ReportStorageFactory { { $project: { _id: 0, - user_count: { $size: '$user_ids' }, - client_id_count: { $size: '$client_ids' }, - sdk: '$sdks' + users: '$user_ids', + clients: '$client_ids', + sdks: '$sdks' } } ]) .toArray(); - console.log(result); return { - ...data, + app_id: data.app_id, + org_id: data.org_id, ...result[0] }; } diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index 5d4ea83eb..17861e495 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -9,5 +9,5 @@ export interface ReportStorageFactory extends AsyncDisposable { reportSdkConnect(data: SdkConnectDocument): Promise; reportSdkDisconnect(data: SdkConnectDocument): Promise; listCurrentConnections(data: InstanceRequest): Promise; - scrapeSdkData(data: InstanceRequest): Promise; + scrapeSdkData(data: InstanceRequest): Promise; } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index edbea68e5..4d76be43a 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -29,18 +29,25 @@ export type SdkConnectDocument = { client_id: string; user_id: string; jwt_exp?: Date; - connect_at?: Date; - disconnect_at?: Date; + connect_at: Date; }; export type InstanceRequest = { app_id: string; org_id: string; - user_count: number; - client_id_count: number; +}; +export type ListCurrentConnections = { + users: string[]; + clients: string[]; sdk: string[]; }; -export type ListCurrentConnectionsResponse = {} & InstanceRequest; + +export type ListCurrentConnectionsResponse = { + users: string[]; + clients: string[]; + sdk: string[]; +} & InstanceRequest; + export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; export interface EmitterEvent { event: K; From 617fc06522c6f97c22e43bf5d75b861f408d51e4 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 12:49:04 +0200 Subject: [PATCH 043/169] removed the disconnect time --- .../service-core/src/routes/endpoints/socket-route.ts | 5 +---- .../service-core/src/routes/endpoints/sync-stream.ts | 10 ++-------- packages/types/src/events.ts | 4 +--- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 88191ec9b..d5085c280 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -177,10 +177,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => close_reason: closeReason ?? 'unknown' }); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { - ...sdkData, - disconnect_at: new Date() - }); + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, sdkData); } } }); diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index df2f72e65..80163b8c2 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -132,10 +132,7 @@ export const syncStreamed = routeDefinition({ } controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { - ...sdkData, - disconnect_at: new Date() - }); + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, sdkData); logger.info(`Sync stream complete`, { ...tracker.getLogMeta(), stream_ms: Date.now() - streamStart, @@ -146,10 +143,7 @@ export const syncStreamed = routeDefinition({ } catch (ex) { controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { - ...sdkData, - disconnect_at: new Date() - }); + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, sdkData); } } }); diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 4d76be43a..e1e265b21 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -19,9 +19,7 @@ export type SdkConnectEventData = { connect_at: Date; } & SdkUserData; -export type SdkDisconnectEventData = { - disconnect_at: Date; -} & SdkUserData; +export type SdkDisconnectEventData = SdkUserData; export type SdkConnectDocument = { sdk: string; From bb9dfd7bf0df4b50b4cd18733095cb005d821dda Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 12:55:16 +0200 Subject: [PATCH 044/169] chnaged disconnect report to use userdata --- .../src/storage/MongoReportStorage.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index c31dc9f5f..bc48246f1 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -3,7 +3,11 @@ import { storage } from '@powersync/service-core'; import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; import { SdkConnectDocument } from './implementation/models.js'; -import { ListCurrentConnections, ListCurrentConnectionsResponse } from '@powersync/service-types/dist/events.js'; +import { + ListCurrentConnections, + ListCurrentConnectionsResponse, + SdkUserData +} from '@powersync/service-types/dist/events.js'; export class MongoReportStorage implements storage.ReportStorageFactory { private readonly client: mongo.MongoClient; @@ -53,7 +57,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } ); } - async reportSdkDisconnect(data: SdkConnectDocument): Promise { + async reportSdkDisconnect(data: SdkUserData): Promise { await this.db.sdk_report_events.findOneAndDelete({ user_id: data.user_id, client_id: data.client_id }); } async listCurrentConnections(data: event_types.InstanceRequest): Promise { From 9a4094eeea9ff09450c08279075ccd936ce4fa40 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 13:02:06 +0200 Subject: [PATCH 045/169] fix --- packages/service-core/src/storage/ReportStorageFactory.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index 17861e495..5a0ee0f5b 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -1,13 +1,14 @@ import { InstanceRequest, ListCurrentConnectionsResponse, - SdkConnectDocument + SdkConnectDocument, + SdkUserData } from '@powersync/service-types/dist/events.js'; // Interface for the ReportStorageFactory export interface ReportStorageFactory extends AsyncDisposable { reportSdkConnect(data: SdkConnectDocument): Promise; - reportSdkDisconnect(data: SdkConnectDocument): Promise; + reportSdkDisconnect(data: SdkUserData): Promise; listCurrentConnections(data: InstanceRequest): Promise; scrapeSdkData(data: InstanceRequest): Promise; } From 644de7158ba87dfa6c3d107a8e27226dab3800cf Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 15:34:19 +0200 Subject: [PATCH 046/169] chnaged sdk scrape and added sdk delete event --- .../src/storage/MongoReportStorage.ts | 8 ++++-- .../src/routes/endpoints/socket-route.ts | 5 +++- .../src/routes/endpoints/sync-stream.ts | 10 +++++-- .../src/storage/ReportStorageFactory.ts | 6 ++-- packages/types/src/events.ts | 28 ++++++++++++------- 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index bc48246f1..115e405fe 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -6,7 +6,7 @@ import { SdkConnectDocument } from './implementation/models.js'; import { ListCurrentConnections, ListCurrentConnectionsResponse, - SdkUserData + SdkDisconnectEventData } from '@powersync/service-types/dist/events.js'; export class MongoReportStorage implements storage.ReportStorageFactory { @@ -18,6 +18,10 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.db = db; } + async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise { + console.log(data); + } + async scrapeSdkData(data: event_types.InstanceRequest): Promise { const result = await this.db.sdk_report_events .aggregate([ @@ -57,7 +61,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } ); } - async reportSdkDisconnect(data: SdkUserData): Promise { + async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { await this.db.sdk_report_events.findOneAndDelete({ user_id: data.user_id, client_id: data.client_id }); } async listCurrentConnections(data: event_types.InstanceRequest): Promise { diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index d5085c280..88191ec9b 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -177,7 +177,10 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => close_reason: closeReason ?? 'unknown' }); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, sdkData); + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { + ...sdkData, + disconnect_at: new Date() + }); } } }); diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 80163b8c2..df2f72e65 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -132,7 +132,10 @@ export const syncStreamed = routeDefinition({ } controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, sdkData); + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { + ...sdkData, + disconnect_at: new Date() + }); logger.info(`Sync stream complete`, { ...tracker.getLogMeta(), stream_ms: Date.now() - streamStart, @@ -143,7 +146,10 @@ export const syncStreamed = routeDefinition({ } catch (ex) { controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, sdkData); + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { + ...sdkData, + disconnect_at: new Date() + }); } } }); diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index 5a0ee0f5b..9c19ddb7b 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -1,14 +1,16 @@ import { + DeleteOldSdkData, InstanceRequest, ListCurrentConnectionsResponse, SdkConnectDocument, - SdkUserData + SdkDisconnectEventData } from '@powersync/service-types/dist/events.js'; // Interface for the ReportStorageFactory export interface ReportStorageFactory extends AsyncDisposable { reportSdkConnect(data: SdkConnectDocument): Promise; - reportSdkDisconnect(data: SdkUserData): Promise; + reportSdkDisconnect(data: SdkDisconnectEventData): Promise; listCurrentConnections(data: InstanceRequest): Promise; scrapeSdkData(data: InstanceRequest): Promise; + deleteOldSdkData(data: DeleteOldSdkData): Promise; } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index e1e265b21..0ed27ae53 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -1,11 +1,13 @@ export enum EmitterEngineEvents { SDK_CONNECT_EVENT = 'sdk-connect-event', - SDK_DISCONNECT_EVENT = 'sdk-disconnect-event' + SDK_DISCONNECT_EVENT = 'sdk-disconnect-event', + SDK_DELETE_OLD = 'sdk-delete-old' } export type SubscribeEvents = { [EmitterEngineEvents.SDK_CONNECT_EVENT]: SdkConnectEventData; [EmitterEngineEvents.SDK_DISCONNECT_EVENT]: SdkDisconnectEventData; + [EmitterEngineEvents.SDK_DELETE_OLD]: DeleteOldSdkData; }; export type SdkUserData = { @@ -15,11 +17,18 @@ export type SdkUserData = { jwt_exp?: Date; }; +export type DeleteOldSdkData = { + interval: number; + timeframe: 'month' | 'week' | 'day'; +}; + export type SdkConnectEventData = { connect_at: Date; } & SdkUserData; -export type SdkDisconnectEventData = SdkUserData; +export type SdkDisconnectEventData = { + disconnect_at: Date; +} & SdkUserData; export type SdkConnectDocument = { sdk: string; @@ -34,17 +43,16 @@ export type InstanceRequest = { app_id: string; org_id: string; }; + export type ListCurrentConnections = { - users: string[]; - clients: string[]; - sdk: string[]; + user_count: number; + client_count: number; + user_per_sdk: { + [sdk_version: string]: number; + }; }; -export type ListCurrentConnectionsResponse = { - users: string[]; - clients: string[]; - sdk: string[]; -} & InstanceRequest; +export type ListCurrentConnectionsResponse = ListCurrentConnections & InstanceRequest; export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; export interface EmitterEvent { From 7d93f69e668e0a70a3aa7f6e4de9510f1efa8931 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 16:48:06 +0200 Subject: [PATCH 047/169] change disconnect mongo update --- .../src/storage/MongoReportStorage.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 115e405fe..56d5233b2 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -62,7 +62,17 @@ export class MongoReportStorage implements storage.ReportStorageFactory { ); } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { - await this.db.sdk_report_events.findOneAndDelete({ user_id: data.user_id, client_id: data.client_id }); + await this.db.sdk_report_events.findOneAndUpdate( + { user_id: data.user_id, client_id: data.client_id }, + { + $set: { + disconnect_at: data.disconnect_at + }, + $unset: { + jwt_exp: '' + } + } + ); } async listCurrentConnections(data: event_types.InstanceRequest): Promise { const result = await this.db.sdk_report_events From efd5600257d76a67780155d6903d88510c709a2c Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 17:04:43 +0200 Subject: [PATCH 048/169] wip --- .../src/storage/MongoReportStorage.ts | 12 +++++----- .../src/routes/endpoints/socket-route.ts | 16 +++++++++---- .../src/routes/endpoints/sync-stream.ts | 24 +++++++++++++------ .../src/storage/ReportStorageFactory.ts | 4 ++-- packages/types/src/events.ts | 16 +++++++++---- 5 files changed, 49 insertions(+), 23 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 56d5233b2..6a0345288 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -2,10 +2,10 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; -import { SdkConnectDocument } from './implementation/models.js'; import { ListCurrentConnections, ListCurrentConnectionsResponse, + SdkConnectEventData, SdkDisconnectEventData } from '@powersync/service-types/dist/events.js'; @@ -50,11 +50,11 @@ export class MongoReportStorage implements storage.ReportStorageFactory { }; } - async reportSdkConnect(data: SdkConnectDocument): Promise { + async reportSdkConnect(data: SdkConnectEventData): Promise { await this.db.sdk_report_events.findOneAndUpdate( - { user_id: data.user_id, client_id: data.client_id }, + { _id: data.id }, { - $set: data + $set: data.data }, { upsert: true @@ -63,10 +63,10 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { await this.db.sdk_report_events.findOneAndUpdate( - { user_id: data.user_id, client_id: data.client_id }, + { _id: data.id }, { $set: { - disconnect_at: data.disconnect_at + disconnect_at: data.data.disconnect_at }, $unset: { jwt_exp: '' diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 88191ec9b..c356041f7 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -1,5 +1,6 @@ import { ErrorCode, errors, schema } from '@powersync/lib-services-framework'; import { RequestParameters } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; import { serialize } from 'bson'; import * as sync from '../../sync/sync-index.js'; @@ -23,6 +24,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => user_agent: context.user_agent }; + const sdkReportId = new bson.ObjectId(); const sdkData: event_types.SdkUserData = { client_id: params.client_id, user_id: context.user_id!, @@ -95,8 +97,11 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, { - ...sdkData, - connect_at: new Date(streamStart) + id: sdkReportId, + data: { + ...sdkData, + connect_at: new Date(streamStart) + } }); const tracker = new sync.RequestTracker(metricsEngine); try { @@ -178,8 +183,11 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => }); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { - ...sdkData, - disconnect_at: new Date() + id: sdkReportId, + data: { + ...sdkData, + disconnect_at: new Date() + } }); } } diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index df2f72e65..2f766ce88 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -3,6 +3,7 @@ import { RequestParameters } from '@powersync/service-sync-rules'; import { Readable } from 'stream'; import * as sync from '../../sync/sync-index.js'; import * as util from '../../util/util-index.js'; +import * as bson from 'bson'; import { authUser } from '../auth.js'; import { routeDefinition } from '../router.js'; @@ -31,7 +32,7 @@ export const syncStreamed = routeDefinition({ client_id: clientId, user_id: payload.context.user_id }; - + const sdkReportId = new bson.ObjectId(); const sdkData: event_types.SdkUserData = { client_id: clientId, user_id: payload.context.user_id!, @@ -67,8 +68,11 @@ export const syncStreamed = routeDefinition({ try { metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, { - ...sdkData, - connect_at: new Date(streamStart) + id: sdkReportId, + data: { + ...sdkData, + connect_at: new Date(streamStart) + } }); const stream = Readable.from( sync.transformToBytesTracked( @@ -133,8 +137,11 @@ export const syncStreamed = routeDefinition({ controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { - ...sdkData, - disconnect_at: new Date() + id: sdkReportId, + data: { + ...sdkData, + disconnect_at: new Date() + } }); logger.info(`Sync stream complete`, { ...tracker.getLogMeta(), @@ -147,8 +154,11 @@ export const syncStreamed = routeDefinition({ controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { - ...sdkData, - disconnect_at: new Date() + id: sdkReportId, + data: { + ...sdkData, + disconnect_at: new Date() + } }); } } diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index 9c19ddb7b..f9d621057 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -2,13 +2,13 @@ import { DeleteOldSdkData, InstanceRequest, ListCurrentConnectionsResponse, - SdkConnectDocument, + SdkConnectEventData, SdkDisconnectEventData } from '@powersync/service-types/dist/events.js'; // Interface for the ReportStorageFactory export interface ReportStorageFactory extends AsyncDisposable { - reportSdkConnect(data: SdkConnectDocument): Promise; + reportSdkConnect(data: SdkConnectEventData): Promise; reportSdkDisconnect(data: SdkDisconnectEventData): Promise; listCurrentConnections(data: InstanceRequest): Promise; scrapeSdkData(data: InstanceRequest): Promise; diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 0ed27ae53..35f93347f 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -1,3 +1,5 @@ +import * as bson from 'bson'; + export enum EmitterEngineEvents { SDK_CONNECT_EVENT = 'sdk-connect-event', SDK_DISCONNECT_EVENT = 'sdk-disconnect-event', @@ -23,12 +25,18 @@ export type DeleteOldSdkData = { }; export type SdkConnectEventData = { - connect_at: Date; -} & SdkUserData; + data: { + connect_at: Date; + } & SdkUserData; + id: bson.ObjectId; +}; export type SdkDisconnectEventData = { - disconnect_at: Date; -} & SdkUserData; + data: { + disconnect_at: Date; + } & SdkUserData; + id: bson.ObjectId; +}; export type SdkConnectDocument = { sdk: string; From 64e397f597c6544b3c8128a57b57795a1210a2f2 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 21 Jul 2025 17:18:42 +0200 Subject: [PATCH 049/169] wip --- .../src/storage/MongoReportStorage.ts | 4 ++-- packages/service-core/src/storage/ReportStorageFactory.ts | 4 ++-- packages/types/src/events.ts | 8 ++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 6a0345288..09777d890 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -5,7 +5,7 @@ import { PowerSyncMongo } from './implementation/db.js'; import { ListCurrentConnections, ListCurrentConnectionsResponse, - SdkConnectEventData, + SdkConnectBucketData, SdkDisconnectEventData } from '@powersync/service-types/dist/events.js'; @@ -50,7 +50,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { }; } - async reportSdkConnect(data: SdkConnectEventData): Promise { + async reportSdkConnect(data: SdkConnectBucketData): Promise { await this.db.sdk_report_events.findOneAndUpdate( { _id: data.id }, { diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index f9d621057..4b766443a 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -2,13 +2,13 @@ import { DeleteOldSdkData, InstanceRequest, ListCurrentConnectionsResponse, - SdkConnectEventData, + SdkConnectBucketData, SdkDisconnectEventData } from '@powersync/service-types/dist/events.js'; // Interface for the ReportStorageFactory export interface ReportStorageFactory extends AsyncDisposable { - reportSdkConnect(data: SdkConnectEventData): Promise; + reportSdkConnect(data: SdkConnectBucketData): Promise; reportSdkDisconnect(data: SdkDisconnectEventData): Promise; listCurrentConnections(data: InstanceRequest): Promise; scrapeSdkData(data: InstanceRequest): Promise; diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 35f93347f..ca7c5e38f 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -31,6 +31,14 @@ export type SdkConnectEventData = { id: bson.ObjectId; }; +export type SdkConnectBucketData = { + data: { + connect_at: Date; + sdk: string; + } & SdkUserData; + id: bson.ObjectId; +}; + export type SdkDisconnectEventData = { data: { disconnect_at: Date; From 87e98c9d4286ace86321dc66d44b74b570e26f22 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 10:36:38 +0200 Subject: [PATCH 050/169] create new doc only if older than a day --- .../src/storage/MongoReportStorage.ts | 37 +++++++++++++------ .../src/routes/endpoints/socket-route.ts | 16 ++------ .../src/routes/endpoints/sync-stream.ts | 23 +++--------- packages/types/src/events.ts | 25 ++++--------- 4 files changed, 42 insertions(+), 59 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 09777d890..f8c0a04b8 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -9,6 +9,19 @@ import { SdkDisconnectEventData } from '@powersync/service-types/dist/events.js'; +function dateFilter(userId: string, clientId: string): mongo.Filter { + const date = new Date(); + const year = date.getFullYear(); + const month = date.getMonth() + 1; + const day = date.getDate(); + return { + user_id: userId, + client_id: clientId, + $gte: new Date(year, month, day, 0, 0, 0), + $lt: new Date(year, month, day + 1, 0, 0, 0) + }; +} + export class MongoReportStorage implements storage.ReportStorageFactory { private readonly client: mongo.MongoClient; public readonly db: PowerSyncMongo; @@ -52,9 +65,12 @@ export class MongoReportStorage implements storage.ReportStorageFactory { async reportSdkConnect(data: SdkConnectBucketData): Promise { await this.db.sdk_report_events.findOneAndUpdate( - { _id: data.id }, + dateFilter(data.user_id, data.client_id!), { - $set: data.data + $set: data, + $unset: { + disconnect_at: '' + } }, { upsert: true @@ -62,17 +78,14 @@ export class MongoReportStorage implements storage.ReportStorageFactory { ); } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { - await this.db.sdk_report_events.findOneAndUpdate( - { _id: data.id }, - { - $set: { - disconnect_at: data.data.disconnect_at - }, - $unset: { - jwt_exp: '' - } + await this.db.sdk_report_events.findOneAndUpdate(dateFilter(data.user_id, data.client_id!), { + $set: { + disconnect_at: data.disconnect_at + }, + $unset: { + jwt_exp: '' } - ); + }); } async listCurrentConnections(data: event_types.InstanceRequest): Promise { const result = await this.db.sdk_report_events diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index c356041f7..88191ec9b 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -1,6 +1,5 @@ import { ErrorCode, errors, schema } from '@powersync/lib-services-framework'; import { RequestParameters } from '@powersync/service-sync-rules'; -import * as bson from 'bson'; import { serialize } from 'bson'; import * as sync from '../../sync/sync-index.js'; @@ -24,7 +23,6 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => user_agent: context.user_agent }; - const sdkReportId = new bson.ObjectId(); const sdkData: event_types.SdkUserData = { client_id: params.client_id, user_id: context.user_id!, @@ -97,11 +95,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, { - id: sdkReportId, - data: { - ...sdkData, - connect_at: new Date(streamStart) - } + ...sdkData, + connect_at: new Date(streamStart) }); const tracker = new sync.RequestTracker(metricsEngine); try { @@ -183,11 +178,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => }); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { - id: sdkReportId, - data: { - ...sdkData, - disconnect_at: new Date() - } + ...sdkData, + disconnect_at: new Date() }); } } diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 2f766ce88..4a6acdb0e 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -3,7 +3,6 @@ import { RequestParameters } from '@powersync/service-sync-rules'; import { Readable } from 'stream'; import * as sync from '../../sync/sync-index.js'; import * as util from '../../util/util-index.js'; -import * as bson from 'bson'; import { authUser } from '../auth.js'; import { routeDefinition } from '../router.js'; @@ -32,7 +31,6 @@ export const syncStreamed = routeDefinition({ client_id: clientId, user_id: payload.context.user_id }; - const sdkReportId = new bson.ObjectId(); const sdkData: event_types.SdkUserData = { client_id: clientId, user_id: payload.context.user_id!, @@ -68,11 +66,8 @@ export const syncStreamed = routeDefinition({ try { metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, { - id: sdkReportId, - data: { - ...sdkData, - connect_at: new Date(streamStart) - } + ...sdkData, + connect_at: new Date(streamStart) }); const stream = Readable.from( sync.transformToBytesTracked( @@ -137,11 +132,8 @@ export const syncStreamed = routeDefinition({ controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { - id: sdkReportId, - data: { - ...sdkData, - disconnect_at: new Date() - } + ...sdkData, + disconnect_at: new Date() }); logger.info(`Sync stream complete`, { ...tracker.getLogMeta(), @@ -154,11 +146,8 @@ export const syncStreamed = routeDefinition({ controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { - id: sdkReportId, - data: { - ...sdkData, - disconnect_at: new Date() - } + ...sdkData, + disconnect_at: new Date() }); } } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index ca7c5e38f..82df8745f 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -1,5 +1,3 @@ -import * as bson from 'bson'; - export enum EmitterEngineEvents { SDK_CONNECT_EVENT = 'sdk-connect-event', SDK_DISCONNECT_EVENT = 'sdk-disconnect-event', @@ -25,26 +23,17 @@ export type DeleteOldSdkData = { }; export type SdkConnectEventData = { - data: { - connect_at: Date; - } & SdkUserData; - id: bson.ObjectId; -}; + connect_at: Date; +} & SdkUserData; export type SdkConnectBucketData = { - data: { - connect_at: Date; - sdk: string; - } & SdkUserData; - id: bson.ObjectId; -}; + connect_at: Date; + sdk: string; +} & SdkUserData; export type SdkDisconnectEventData = { - data: { - disconnect_at: Date; - } & SdkUserData; - id: bson.ObjectId; -}; + disconnect_at: Date; +} & SdkUserData; export type SdkConnectDocument = { sdk: string; From 1ee1d390fbe57c074d88316d1aa8a71ea5fb3102 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 10:48:53 +0200 Subject: [PATCH 051/169] wip --- .../src/storage/MongoReportStorage.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index f8c0a04b8..873ea7e71 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -17,8 +17,10 @@ function dateFilter(userId: string, clientId: string): mongo.Filter Date: Tue, 22 Jul 2025 10:58:21 +0200 Subject: [PATCH 052/169] wip --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 873ea7e71..e858bf190 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -14,7 +14,7 @@ function dateFilter(userId: string, clientId: string): mongo.Filter Date: Tue, 22 Jul 2025 11:04:45 +0200 Subject: [PATCH 053/169] duh --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index e858bf190..ea80a0560 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -12,7 +12,7 @@ import { function dateFilter(userId: string, clientId: string): mongo.Filter { const date = new Date(); const year = date.getFullYear(); - const month = date.getMonth() + 1; + const month = date.getMonth(); const day = date.getDate(); const query = { user_id: userId, From b21770fc996e5ddfcf4906076eae09981d4b20e4 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 14:21:18 +0200 Subject: [PATCH 054/169] scrape data --- .../src/storage/MongoReportStorage.ts | 143 +++++++++++++----- .../src/routes/endpoints/socket-route.ts | 2 +- .../src/routes/endpoints/sync-stream.ts | 2 +- .../src/storage/ReportStorageFactory.ts | 18 +-- packages/types/src/events.ts | 12 +- 5 files changed, 124 insertions(+), 53 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index ea80a0560..20ceaf675 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -2,28 +2,71 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; -import { - ListCurrentConnections, - ListCurrentConnectionsResponse, - SdkConnectBucketData, - SdkDisconnectEventData -} from '@powersync/service-types/dist/events.js'; -function dateFilter(userId: string, clientId: string): mongo.Filter { - const date = new Date(); +function parseDate(date: Date) { const year = date.getFullYear(); const month = date.getMonth(); - const day = date.getDate(); - const query = { + const today = date.getDate(); + const day = date.getDay(); + return { + year, + month, + today, + day + }; +} +function updateDocFilter(userId: string, clientId: string): mongo.Filter { + const { year, month, today } = parseDate(new Date()); + const nextDay = today + 1; + return { user_id: userId, client_id: clientId, connect_at: { - $gte: new Date(year, month, day, 0, 0, 0), - $lt: new Date(year, month, day + 1, 0, 0, 0) + $gte: new Date(year, month, today, 0, 0, 0), + $lt: new Date(year, month, nextDay, 0, 0, 0) } }; - console.log({ query, date }); - return query; +} + +function timeSpan(timeframe: event_types.TimeFrames): mongo.Filter { + const date = new Date(); + const { year, month, day, today } = parseDate(date); + switch (timeframe) { + case event_types.TimeFrames.MONTH: { + // Cron should run the first day of the new month, this then retrieves from the 1st to the last day of the month + const thisMonth = month; + const nextMonth = month == 11 ? 0 : month + 1; + return { + connect_at: { $gte: new Date(year, thisMonth, 1, 0, 0, 0), $lte: new Date(year, nextMonth, 1, 0, 0, 0) } + }; + } + case event_types.TimeFrames.WEEK: { + // Back tracks the date to the previous week Monday to Sunday + const daysToSunday = 0 - day; + const weekEndDate = new Date(date); + weekEndDate.setDate(weekEndDate.getDate() + daysToSunday); + const weekStartDate = new Date(weekEndDate); + weekStartDate.setDate(weekStartDate.getDate() - 6); + const weekStart = parseDate(weekStartDate); + const weekEnd = parseDate(weekEndDate); + return { + connect_at: { + $gte: new Date(weekStart.year, weekStart.month, weekStart.today, 0, 0, 0), + $lte: new Date(weekEnd.year, weekEnd.month, weekEnd.today, 0, 0, 0) + } + }; + } + default: { + // Start from today to just before tomorrow + const tomorrow = today + 1; + return { + connect_at: { + $gte: new Date(date.getFullYear(), date.getMonth(), today, 0, 0, 0), + $lte: new Date(date.getFullYear(), date.getMonth(), tomorrow, 0, 0, 0) + } + }; + } + } } export class MongoReportStorage implements storage.ReportStorageFactory { @@ -39,37 +82,63 @@ export class MongoReportStorage implements storage.ReportStorageFactory { console.log(data); } - async scrapeSdkData(data: event_types.InstanceRequest): Promise { + async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { const result = await this.db.sdk_report_events - .aggregate([ + .aggregate([ { - $group: { - _id: null, - user_ids: { $addToSet: '$user_id' }, - client_ids: { $addToSet: '$client_id' }, - sdks: { $addToSet: '$sdk' } + $match: { + connect_at: timeSpan(data.scrape_time) } }, { - $project: { - _id: 0, - users: '$user_ids', - clients: '$client_ids', - sdks: '$sdks' + $facet: { + unique_user_ids: [ + { + $group: { + _id: '$user_id' + } + }, + { + $count: 'count' + } + ], + unique_user_sdk: [ + { + $group: { + _id: { + user_id: '$user_id', + sdk: '$sdk' + } + } + }, + { + $count: 'count' + } + ], + unique_user_client: [ + { + $group: { + _id: { + user_id: '$user_id', + client_id: '$client_id' + } + } + }, + { + $count: 'count' + } + ] } } ]) .toArray(); - return { - app_id: data.app_id, - org_id: data.org_id, - ...result[0] - }; + console.log(result[0]); + return result[0] as event_types.ListCurrentConnectionsResponse; } - async reportSdkConnect(data: SdkConnectBucketData): Promise { + async reportSdkConnect(data: event_types.SdkConnectBucketData): Promise { await this.db.sdk_report_events.findOneAndUpdate( - dateFilter(data.user_id, data.client_id!), + updateDocFilter(data.user_id, data.client_id!), { $set: data, $unset: { @@ -81,8 +150,8 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } ); } - async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { - await this.db.sdk_report_events.findOneAndUpdate(dateFilter(data.user_id, data.client_id!), { + async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { + await this.db.sdk_report_events.findOneAndUpdate(updateDocFilter(data.user_id, data.client_id!), { $set: { disconnect_at: data.disconnect_at }, @@ -91,9 +160,9 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } }); } - async listCurrentConnections(data: event_types.InstanceRequest): Promise { + async listCurrentConnections(data: event_types.InstanceRequest): Promise { const result = await this.db.sdk_report_events - .aggregate([ + .aggregate([ { $group: { _id: null, diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 88191ec9b..5b77540ec 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -27,7 +27,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => client_id: params.client_id, user_id: context.user_id!, user_agent: context.user_agent, - jwt_exp: context.token_payload?.exp ? new Date(context.token_payload.exp) : undefined + jwt_exp: context.token_payload?.exp ? new Date(context.token_payload.exp * 1000) : undefined }; const streamStart = Date.now(); diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 4a6acdb0e..ed3cb7853 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -35,7 +35,7 @@ export const syncStreamed = routeDefinition({ client_id: clientId, user_id: payload.context.user_id!, user_agent: userAgent as string, - jwt_exp: token_payload?.exp ? new Date(token_payload?.exp) : undefined + jwt_exp: token_payload?.exp ? new Date(token_payload?.exp * 1000) : undefined }; if (routerEngine.closed) { diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index 4b766443a..07c18382d 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -1,16 +1,10 @@ -import { - DeleteOldSdkData, - InstanceRequest, - ListCurrentConnectionsResponse, - SdkConnectBucketData, - SdkDisconnectEventData -} from '@powersync/service-types/dist/events.js'; +import { event_types } from '@powersync/service-types'; // Interface for the ReportStorageFactory export interface ReportStorageFactory extends AsyncDisposable { - reportSdkConnect(data: SdkConnectBucketData): Promise; - reportSdkDisconnect(data: SdkDisconnectEventData): Promise; - listCurrentConnections(data: InstanceRequest): Promise; - scrapeSdkData(data: InstanceRequest): Promise; - deleteOldSdkData(data: DeleteOldSdkData): Promise; + reportSdkConnect(data: event_types.SdkConnectBucketData): Promise; + reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise; + listCurrentConnections(data: event_types.InstanceRequest): Promise; + scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise; + deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise; } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 82df8745f..31bbaf311 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -4,6 +4,12 @@ export enum EmitterEngineEvents { SDK_DELETE_OLD = 'sdk-delete-old' } +export enum TimeFrames { + DAY = 'day', + WEEK = 'week', + MONTH = 'month' +} + export type SubscribeEvents = { [EmitterEngineEvents.SDK_CONNECT_EVENT]: SdkConnectEventData; [EmitterEngineEvents.SDK_DISCONNECT_EVENT]: SdkDisconnectEventData; @@ -19,7 +25,7 @@ export type SdkUserData = { export type DeleteOldSdkData = { interval: number; - timeframe: 'month' | 'week' | 'day'; + timeframe: TimeFrames; }; export type SdkConnectEventData = { @@ -56,7 +62,9 @@ export type ListCurrentConnections = { [sdk_version: string]: number; }; }; - +export type ScrapeSdkDataRequest = { + scrape_time: TimeFrames; +}; export type ListCurrentConnectionsResponse = ListCurrentConnections & InstanceRequest; export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; From 06a1859fb3d5d01431a12c1969db82e5883675d7 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 14:49:56 +0200 Subject: [PATCH 055/169] wip --- .../src/storage/MongoReportStorage.ts | 4 ++-- packages/types/src/events.ts | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 20ceaf675..a14822b40 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -32,7 +32,7 @@ function timeSpan(timeframe: event_types.TimeFrames): mongo.Filter Date: Tue, 22 Jul 2025 15:05:14 +0200 Subject: [PATCH 056/169] timespan logging --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index a14822b40..30fa23537 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -83,11 +83,13 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { + const timespanFilter = timeSpan(data.scrape_time); + console.log(timespanFilter); const result = await this.db.sdk_report_events .aggregate([ { $match: { - connect_at: timeSpan(data.scrape_time) + connect_at: timespanFilter } }, { @@ -132,7 +134,6 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } ]) .toArray(); - console.log(result[0]); return result[0] as event_types.ListCurrentConnectionsResponse; } From 453ec7267405f89cb93165f0d8446c171623a78e Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 15:17:12 +0200 Subject: [PATCH 057/169] fix day query bug --- .../src/storage/MongoReportStorage.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 30fa23537..0ce65f947 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -58,11 +58,11 @@ function timeSpan(timeframe: event_types.TimeFrames): mongo.Filter Date: Tue, 22 Jul 2025 15:26:54 +0200 Subject: [PATCH 058/169] remove facet --- .../src/storage/MongoReportStorage.ts | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 0ce65f947..8be4d7c06 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -91,47 +91,47 @@ export class MongoReportStorage implements storage.ReportStorageFactory { $match: { connect_at: timespanFilter } - }, - { - $facet: { - unique_user_ids: [ - { - $group: { - _id: '$user_id' - } - }, - { - $count: 'count' - } - ], - unique_user_sdk: [ - { - $group: { - _id: { - user_id: '$user_id', - sdk: '$sdk' - } - } - }, - { - $count: 'count' - } - ], - unique_user_client: [ - { - $group: { - _id: { - user_id: '$user_id', - client_id: '$client_id' - } - } - }, - { - $count: 'count' - } - ] - } } + // { + // $facet: { + // unique_user_ids: [ + // { + // $group: { + // _id: '$user_id' + // } + // }, + // { + // $count: 'count' + // } + // ], + // unique_user_sdk: [ + // { + // $group: { + // _id: { + // user_id: '$user_id', + // sdk: '$sdk' + // } + // } + // }, + // { + // $count: 'count' + // } + // ], + // unique_user_client: [ + // { + // $group: { + // _id: { + // user_id: '$user_id', + // client_id: '$client_id' + // } + // } + // }, + // { + // $count: 'count' + // } + // ] + // } + // } ]) .toArray(); return result[0] as event_types.ListCurrentConnectionsResponse; From b123e437b2e2c235b11af4f000dba08785004505 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 15:54:56 +0200 Subject: [PATCH 059/169] dates --- .../src/storage/MongoReportStorage.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 8be4d7c06..267cc07e7 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -22,8 +22,8 @@ function updateDocFilter(userId: string, clientId: string): mongo.Filter Date: Tue, 22 Jul 2025 16:04:51 +0200 Subject: [PATCH 060/169] wip --- .../src/storage/MongoReportStorage.ts | 104 +++++++++--------- 1 file changed, 49 insertions(+), 55 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 267cc07e7..463d203b5 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -15,20 +15,20 @@ function parseDate(date: Date) { day }; } -function updateDocFilter(userId: string, clientId: string): mongo.Filter { +function updateDocFilter(userId: string, clientId: string) { const { year, month, today } = parseDate(new Date()); const nextDay = today + 1; return { user_id: userId, client_id: clientId, connect_at: { - $gte: new Date(`${year}-${month}-${today}`), - $lt: new Date(`${year}-${month}-${nextDay}`) + $gte: new Date(year, month, today), + $lt: new Date(year, month, nextDay) } }; } -function timeSpan(timeframe: event_types.TimeFrames): mongo.Filter { +function timeSpan(timeframe: event_types.TimeFrames) { const date = new Date(); const { year, month, day, today } = parseDate(date); switch (timeframe) { @@ -36,9 +36,7 @@ function timeSpan(timeframe: event_types.TimeFrames): mongo.Filter Date: Tue, 22 Jul 2025 16:06:12 +0200 Subject: [PATCH 061/169] wip --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 463d203b5..6b8b33f65 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -85,7 +85,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { $match: { connect_at: timespanFilter } - } + }, { $facet: { unique_user_ids: [ From d56753e5454856cb2ec6a8e23c1ecedb95566a34 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 16:19:41 +0200 Subject: [PATCH 062/169] unique sdks --- .../src/storage/MongoReportStorage.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 6b8b33f65..380118bf1 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -88,7 +88,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { }, { $facet: { - unique_user_ids: [ + unique_users: [ { $group: { _id: '$user_id' @@ -123,7 +123,20 @@ export class MongoReportStorage implements storage.ReportStorageFactory { { $count: 'count' } + ], + unique_sdks: [ + { + $group: { + _id: '$sdk' + } + } ] + }, + $project: { + unique_users_count: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, + unique_user_sdk_count: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, + unique_user_client_count: { $ifNull: [{ $arrayElemAt: ['$unique_user_client.count', 0] }, 0] }, + sdk_versions: '$unique_sdks._id' } } ]) From 5f59578d3dc52becf7989e990a13e2b90d4928d8 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 16:25:52 +0200 Subject: [PATCH 063/169] wip --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 380118bf1..55f442eb3 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -131,7 +131,9 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } } ] - }, + } + }, + { $project: { unique_users_count: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, unique_user_sdk_count: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, From bb742da3ee9241087e396372a8b7f9e600cf9c6f Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 16:47:38 +0200 Subject: [PATCH 064/169] sdk --- .../src/storage/MongoReportStorage.ts | 13 ++++++++++--- packages/types/src/events.ts | 9 +++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 55f442eb3..c6875d7ea 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -124,10 +124,17 @@ export class MongoReportStorage implements storage.ReportStorageFactory { $count: 'count' } ], - unique_sdks: [ + sdk_versions_array: [ { $group: { - _id: '$sdk' + _id: '$sdk', + count: { $sum: 1 } + } + }, + { + $project: { + k: '$_id', + v: '$count' } } ] @@ -138,7 +145,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { unique_users_count: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, unique_user_sdk_count: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, unique_user_client_count: { $ifNull: [{ $arrayElemAt: ['$unique_user_client.count', 0] }, 0] }, - sdk_versions: '$unique_sdks._id' + sdk_versions: { $arrayToObject: '$sdk_versions_array' } } } ]) diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 69b67a744..d45d74588 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -51,16 +51,17 @@ export type InstanceRequest = { }; export type ListCurrentConnections = { - user_count: number; - client_count: number; - user_per_sdk: { + unique_user_count: number; + unique_user_client_count: number; + unique_user_sdk_count: number; + sdk_versions: { [sdk_version: string]: number; }; }; export type ScrapeSdkDataRequest = { scrape_time: TimeFrames; }; -export type ListCurrentConnectionsResponse = ListCurrentConnections & InstanceRequest; +export type ListCurrentConnectionsResponse = ListCurrentConnections; export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; export interface EmitterEvent { From 28efea4cf501257bf44c4e6873b63d87d20544fd Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 16:51:17 +0200 Subject: [PATCH 065/169] sdk --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index c6875d7ea..098ae8586 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -199,8 +199,6 @@ export class MongoReportStorage implements storage.ReportStorageFactory { ]) .toArray(); return { - app_id: data.app_id, - org_id: data.org_id, ...result[0] }; } From a4645b9017bf5ec1fa9f36872cf04350611feaa1 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 16:56:13 +0200 Subject: [PATCH 066/169] wip --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 098ae8586..5ba76d679 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -145,7 +145,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { unique_users_count: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, unique_user_sdk_count: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, unique_user_client_count: { $ifNull: [{ $arrayElemAt: ['$unique_user_client.count', 0] }, 0] }, - sdk_versions: { $arrayToObject: '$sdk_versions_array' } + sdk_versions: '$sdk_versions_array' } } ]) From bc1819832ab951de23ddc559910d0d54ecdb69ec Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 22 Jul 2025 17:00:28 +0200 Subject: [PATCH 067/169] wip --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 5ba76d679..0ec3e744f 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -133,6 +133,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { }, { $project: { + _id: 0, k: '$_id', v: '$count' } @@ -145,7 +146,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { unique_users_count: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, unique_user_sdk_count: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, unique_user_client_count: { $ifNull: [{ $arrayElemAt: ['$unique_user_client.count', 0] }, 0] }, - sdk_versions: '$sdk_versions_array' + sdk_versions: { $arrayToObject: '$sdk_versions_array' } } } ]) From 6e1c1ef2ec784471038eb0a8824f74e2b8e890f2 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 10:03:52 +0200 Subject: [PATCH 068/169] list connections and some helpers --- .../src/storage/MongoReportStorage.ts | 130 ++++++++++++++---- .../src/storage/ReportStorageFactory.ts | 4 +- packages/types/src/events.ts | 25 ++-- 3 files changed, 122 insertions(+), 37 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 0ec3e744f..74827c4cc 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -28,15 +28,23 @@ function updateDocFilter(userId: string, clientId: string) { }; } -function timeSpan(timeframe: event_types.TimeFrames) { +function parseMonthInterval(currentMonth: number, interval: number) { + const readableMonth = currentMonth + 1; + const difference = readableMonth - interval; + if (difference < 0) { + return 12 + difference - 1; + } + return difference - 1; +} + +function timeSpan(period: event_types.TimeFrames, timeframe: number = 1): mongo.Filter { const date = new Date(); const { year, month, day, today } = parseDate(date); - switch (timeframe) { + switch (period) { case 'month': { - // Cron should run the first day of the new month, this then retrieves from the 1st to the last day of the month const thisMonth = month; - const nextMonth = month == 11 ? 0 : month + 1; - return { $gte: new Date(year, thisMonth), $lte: new Date(year, nextMonth) }; + const previousMonth = parseMonthInterval(thisMonth, timeframe); + return { $gte: new Date(year, thisMonth), $lt: new Date(year, previousMonth) }; } case 'week': { // Back tracks the date to the previous week Monday to Sunday @@ -44,7 +52,7 @@ function timeSpan(timeframe: event_types.TimeFrames) { const weekEndDate = new Date(date); weekEndDate.setDate(weekEndDate.getDate() + daysToSunday); const weekStartDate = new Date(weekEndDate); - weekStartDate.setDate(weekStartDate.getDate() - 6); + weekStartDate.setDate(weekStartDate.getDate() - 6 * timeframe); const weekStart = parseDate(weekStartDate); const weekEnd = parseDate(weekEndDate); return { @@ -52,12 +60,19 @@ function timeSpan(timeframe: event_types.TimeFrames) { $lte: new Date(weekEnd.year, weekEnd.month, weekEnd.today) }; } + case 'hour': { + // Get the last hour from the current time + const previousHour = date.getHours() - timeframe; + return { + $gte: new Date(year, month, today, previousHour), + $lt: new Date(year, month, today, date.getHours()) + }; + } default: { // Start from today to just before tomorrow - const nextDay = today + 1; return { $gte: new Date(year, month, today), - $lt: new Date(year, month, nextDay) + $lte: new Date(year, month, today - timeframe) }; } } @@ -73,12 +88,15 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise { - console.log(data); + const { period, timeframe } = data; + await this.db.sdk_report_events.deleteMany({ + connect_at: timeSpan(period, timeframe), + $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }] + }); } async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { - const timespanFilter = timeSpan(data.scrape_time); - console.log(timespanFilter); + const timespanFilter = timeSpan(data.period); const result = await this.db.sdk_report_events .aggregate([ { @@ -143,9 +161,9 @@ export class MongoReportStorage implements storage.ReportStorageFactory { }, { $project: { - unique_users_count: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, - unique_user_sdk_count: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, - unique_user_client_count: { $ifNull: [{ $arrayElemAt: ['$unique_user_client.count', 0] }, 0] }, + users: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, + user_sdk: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, + client_user: { $ifNull: [{ $arrayElemAt: ['$unique_user_client.count', 0] }, 0] }, sdk_versions: { $arrayToObject: '$sdk_versions_array' } } } @@ -178,30 +196,86 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } }); } - async listCurrentConnections(data: event_types.InstanceRequest): Promise { + async listCurrentConnections( + data: event_types.ListCurrentConnectionsRequest + ): Promise { + const timespanFilter = data.period ? { connect_at: timeSpan(data.period) } : undefined; + console.log({ timespanFilter }); const result = await this.db.sdk_report_events - .aggregate([ + .aggregate([ { - $group: { - _id: null, - user_ids: { $addToSet: '$user_id' }, - client_ids: { $addToSet: '$client_id' }, - sdks: { $addToSet: '$sdk' } + $match: { + disconnect_at: { $exists: false }, + jwt_exp: { $gt: new Date() }, + ...timespanFilter + } + }, + { + $facet: { + unique_users: [ + { + $group: { + _id: '$user_id' + } + }, + { + $count: 'count' + } + ], + unique_user_sdk: [ + { + $group: { + _id: { + user_id: '$user_id', + sdk: '$sdk' + } + } + }, + { + $count: 'count' + } + ], + unique_user_client: [ + { + $group: { + _id: { + user_id: '$user_id', + client_id: '$client_id' + } + } + }, + { + $count: 'count' + } + ], + sdk_versions_array: [ + { + $group: { + _id: '$sdk', + count: { $sum: 1 } + } + }, + { + $project: { + _id: 0, + k: '$_id', + v: '$count' + } + } + ] } }, { $project: { - _id: 0, - users: '$user_ids', - clients: '$client_ids', - sdks: '$sdks' + users: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, + user_sdk: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, + client_user: { $ifNull: [{ $arrayElemAt: ['$unique_user_client.count', 0] }, 0] }, + sdk_versions: { $arrayToObject: '$sdk_versions_array' } } } ]) .toArray(); - return { - ...result[0] - }; + return result[0] as event_types.ListCurrentConnectionsResponse; } async [Symbol.asyncDispose]() { diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index 07c18382d..f069c1307 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -4,7 +4,9 @@ import { event_types } from '@powersync/service-types'; export interface ReportStorageFactory extends AsyncDisposable { reportSdkConnect(data: event_types.SdkConnectBucketData): Promise; reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise; - listCurrentConnections(data: event_types.InstanceRequest): Promise; + listCurrentConnections( + data: event_types.ListCurrentConnectionsRequest + ): Promise; scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise; deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise; } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index d45d74588..111537dd3 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -4,7 +4,7 @@ export enum EmitterEngineEvents { SDK_DELETE_OLD = 'sdk-delete-old' } -export type TimeFrames = 'day' | 'week' | 'month'; +export type TimeFrames = 'hour' | 'day' | 'week' | 'month'; export type SubscribeEvents = { [EmitterEngineEvents.SDK_CONNECT_EVENT]: SdkConnectEventData; [EmitterEngineEvents.SDK_DISCONNECT_EVENT]: SdkDisconnectEventData; @@ -19,8 +19,8 @@ export type SdkUserData = { }; export type DeleteOldSdkData = { - interval: number; - timeframe: TimeFrames; + timeframe: number; + period: TimeFrames; }; export type SdkConnectEventData = { @@ -43,6 +43,7 @@ export type SdkConnectDocument = { user_id: string; jwt_exp?: Date; connect_at: Date; + disconnect_at?: Date; }; export type InstanceRequest = { @@ -51,16 +52,24 @@ export type InstanceRequest = { }; export type ListCurrentConnections = { - unique_user_count: number; - unique_user_client_count: number; - unique_user_sdk_count: number; - sdk_versions: { + users: number; + // Grouping user_id and sdk versions + user_sdk: number; + // Grouping user_id and client_ids + client_user: number; + // Counts of used sdk versions ( outdated sdks will have the key of unknown) + sdks: { [sdk_version: string]: number; }; }; export type ScrapeSdkDataRequest = { - scrape_time: TimeFrames; + period: TimeFrames; }; + +export type ListCurrentConnectionsRequest = { + period?: TimeFrames; +}; + export type ListCurrentConnectionsResponse = ListCurrentConnections; export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; From 83972c619f41f7e7f812588641f1fdefb5fdc8ed Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 10:42:13 +0200 Subject: [PATCH 069/169] corrections timeframes --- .../src/storage/MongoReportStorage.ts | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 74827c4cc..ef637cc15 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -28,36 +28,24 @@ function updateDocFilter(userId: string, clientId: string) { }; } -function parseMonthInterval(currentMonth: number, interval: number) { - const readableMonth = currentMonth + 1; - const difference = readableMonth - interval; - if (difference < 0) { - return 12 + difference - 1; - } - return difference - 1; -} - function timeSpan(period: event_types.TimeFrames, timeframe: number = 1): mongo.Filter { const date = new Date(); - const { year, month, day, today } = parseDate(date); + const { year, month, today } = parseDate(date); switch (period) { case 'month': { - const thisMonth = month; - const previousMonth = parseMonthInterval(thisMonth, timeframe); - return { $gte: new Date(year, thisMonth), $lt: new Date(year, previousMonth) }; + return { $lte: date, $gte: new Date(year, date.getMonth() - timeframe) }; } case 'week': { // Back tracks the date to the previous week Monday to Sunday - const daysToSunday = 0 - day; - const weekEndDate = new Date(date); - weekEndDate.setDate(weekEndDate.getDate() + daysToSunday); - const weekStartDate = new Date(weekEndDate); + // const daysToSunday = 0 - day; + // const weekEndDate = new Date(date); + // weekEndDate.setDate(weekEndDate.getDate() + daysToSunday); + const weekStartDate = new Date(date); weekStartDate.setDate(weekStartDate.getDate() - 6 * timeframe); const weekStart = parseDate(weekStartDate); - const weekEnd = parseDate(weekEndDate); return { $gte: new Date(weekStart.year, weekStart.month, weekStart.today), - $lte: new Date(weekEnd.year, weekEnd.month, weekEnd.today) + $lte: new Date(year, month, today) }; } case 'hour': { @@ -71,8 +59,8 @@ function timeSpan(period: event_types.TimeFrames, timeframe: number = 1): mongo. default: { // Start from today to just before tomorrow return { - $gte: new Date(year, month, today), - $lte: new Date(year, month, today - timeframe) + $lte: new Date(year, month, today), + $gte: new Date(year, month, today - timeframe) }; } } From fcc0468e2305b765de4beb6c0b9941bbe1b0759b Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 10:47:14 +0200 Subject: [PATCH 070/169] testing --- .../src/storage/MongoReportStorage.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index ef637cc15..e599ac131 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -36,16 +36,12 @@ function timeSpan(period: event_types.TimeFrames, timeframe: number = 1): mongo. return { $lte: date, $gte: new Date(year, date.getMonth() - timeframe) }; } case 'week': { - // Back tracks the date to the previous week Monday to Sunday - // const daysToSunday = 0 - day; - // const weekEndDate = new Date(date); - // weekEndDate.setDate(weekEndDate.getDate() + daysToSunday); const weekStartDate = new Date(date); weekStartDate.setDate(weekStartDate.getDate() - 6 * timeframe); const weekStart = parseDate(weekStartDate); return { $gte: new Date(weekStart.year, weekStart.month, weekStart.today), - $lte: new Date(year, month, today) + $lte: date }; } case 'hour': { @@ -59,7 +55,7 @@ function timeSpan(period: event_types.TimeFrames, timeframe: number = 1): mongo. default: { // Start from today to just before tomorrow return { - $lte: new Date(year, month, today), + $lte: date, $gte: new Date(year, month, today - timeframe) }; } From 486a0941d98115a03a596b4431bc625f68ef9ab6 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 11:34:24 +0200 Subject: [PATCH 071/169] scrapeSdk interval --- .../src/storage/MongoReportStorage.ts | 210 +++++++----------- .../src/storage/ReportStorageFactory.ts | 1 - packages/types/src/events.ts | 3 +- 3 files changed, 83 insertions(+), 131 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index e599ac131..7a9696907 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -2,6 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; +import { logger } from '@powersync/lib-services-framework'; function parseDate(date: Date) { const year = date.getFullYear(); @@ -71,16 +72,90 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.db = db; } + private sdkFacetPipeline() { + return { + $facet: { + unique_users: [ + { + $group: { + _id: '$user_id' + } + }, + { + $count: 'count' + } + ], + unique_user_sdk: [ + { + $group: { + _id: { + user_id: '$user_id', + sdk: '$sdk' + } + } + }, + { + $count: 'count' + } + ], + unique_user_client: [ + { + $group: { + _id: { + user_id: '$user_id', + client_id: '$client_id' + } + } + }, + { + $count: 'count' + } + ], + sdk_versions_array: [ + { + $group: { + _id: '$sdk', + count: { $sum: 1 } + } + }, + { + $project: { + _id: 0, + k: '$_id', + v: '$count' + } + } + ] + } + }; + } + + private sdkProjectPipeline() { + return { + $project: { + users: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, + user_sdk: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, + client_user: { $ifNull: [{ $arrayElemAt: ['$unique_user_client.count', 0] }, 0] }, + sdk_versions: { $arrayToObject: '$sdk_versions_array' } + } + }; + } + async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise { const { period, timeframe } = data; - await this.db.sdk_report_events.deleteMany({ + const result = await this.db.sdk_report_events.deleteMany({ connect_at: timeSpan(period, timeframe), $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }] }); + console.log(result); + if (result.deletedCount > 0) { + logger.info(`TTL: ${result.deletedCount} documents have been removed from sdk_report_events collection`); + } } async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { - const timespanFilter = timeSpan(data.period); + const timespanFilter = timeSpan(data.period, data.interval); + console.log({ timespanFilter }); const result = await this.db.sdk_report_events .aggregate([ { @@ -88,69 +163,8 @@ export class MongoReportStorage implements storage.ReportStorageFactory { connect_at: timespanFilter } }, - { - $facet: { - unique_users: [ - { - $group: { - _id: '$user_id' - } - }, - { - $count: 'count' - } - ], - unique_user_sdk: [ - { - $group: { - _id: { - user_id: '$user_id', - sdk: '$sdk' - } - } - }, - { - $count: 'count' - } - ], - unique_user_client: [ - { - $group: { - _id: { - user_id: '$user_id', - client_id: '$client_id' - } - } - }, - { - $count: 'count' - } - ], - sdk_versions_array: [ - { - $group: { - _id: '$sdk', - count: { $sum: 1 } - } - }, - { - $project: { - _id: 0, - k: '$_id', - v: '$count' - } - } - ] - } - }, - { - $project: { - users: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, - user_sdk: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, - client_user: { $ifNull: [{ $arrayElemAt: ['$unique_user_client.count', 0] }, 0] }, - sdk_versions: { $arrayToObject: '$sdk_versions_array' } - } - } + this.sdkFacetPipeline(), + this.sdkProjectPipeline() ]) .toArray(); return result[0] as event_types.ListCurrentConnectionsResponse; @@ -184,7 +198,6 @@ export class MongoReportStorage implements storage.ReportStorageFactory { data: event_types.ListCurrentConnectionsRequest ): Promise { const timespanFilter = data.period ? { connect_at: timeSpan(data.period) } : undefined; - console.log({ timespanFilter }); const result = await this.db.sdk_report_events .aggregate([ { @@ -194,69 +207,8 @@ export class MongoReportStorage implements storage.ReportStorageFactory { ...timespanFilter } }, - { - $facet: { - unique_users: [ - { - $group: { - _id: '$user_id' - } - }, - { - $count: 'count' - } - ], - unique_user_sdk: [ - { - $group: { - _id: { - user_id: '$user_id', - sdk: '$sdk' - } - } - }, - { - $count: 'count' - } - ], - unique_user_client: [ - { - $group: { - _id: { - user_id: '$user_id', - client_id: '$client_id' - } - } - }, - { - $count: 'count' - } - ], - sdk_versions_array: [ - { - $group: { - _id: '$sdk', - count: { $sum: 1 } - } - }, - { - $project: { - _id: 0, - k: '$_id', - v: '$count' - } - } - ] - } - }, - { - $project: { - users: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, - user_sdk: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, - client_user: { $ifNull: [{ $arrayElemAt: ['$unique_user_client.count', 0] }, 0] }, - sdk_versions: { $arrayToObject: '$sdk_versions_array' } - } - } + this.sdkFacetPipeline(), + this.sdkProjectPipeline() ]) .toArray(); return result[0] as event_types.ListCurrentConnectionsResponse; diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index f069c1307..bb92dc703 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -1,6 +1,5 @@ import { event_types } from '@powersync/service-types'; -// Interface for the ReportStorageFactory export interface ReportStorageFactory extends AsyncDisposable { reportSdkConnect(data: event_types.SdkConnectBucketData): Promise; reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise; diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 111537dd3..a98b53a09 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -64,10 +64,11 @@ export type ListCurrentConnections = { }; export type ScrapeSdkDataRequest = { period: TimeFrames; + interval?: number; }; export type ListCurrentConnectionsRequest = { - period?: TimeFrames; + period?: 'day'; }; export type ListCurrentConnectionsResponse = ListCurrentConnections; From 58fdb79639dd1882efca4ca95dfbe39a77652e38 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 12:39:21 +0200 Subject: [PATCH 072/169] change to response --- .../src/storage/MongoReportStorage.ts | 47 +++++-------------- .../src/storage/ReportStorageFactory.ts | 6 +-- packages/types/src/events.ts | 29 ++++-------- 3 files changed, 23 insertions(+), 59 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 7a9696907..0d41bfc34 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -85,44 +85,21 @@ export class MongoReportStorage implements storage.ReportStorageFactory { $count: 'count' } ], - unique_user_sdk: [ - { - $group: { - _id: { - user_id: '$user_id', - sdk: '$sdk' - } - } - }, - { - $count: 'count' - } - ], - unique_user_client: [ - { - $group: { - _id: { - user_id: '$user_id', - client_id: '$client_id' - } - } - }, - { - $count: 'count' - } - ], sdk_versions_array: [ { $group: { _id: '$sdk', - count: { $sum: 1 } + total: { $sum: 1 }, + client_ids: { $addToSet: '$client_id' }, + user_ids: { $addToSet: '$user_id' } } }, { $project: { _id: 0, - k: '$_id', - v: '$count' + sdk: '$_id', + users: { $size: '$user_ids' }, + clients: { $size: '$client_ids' } } } ] @@ -134,9 +111,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { return { $project: { users: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, - user_sdk: { $ifNull: [{ $arrayElemAt: ['$unique_user_sdk.count', 0] }, 0] }, - client_user: { $ifNull: [{ $arrayElemAt: ['$unique_user_client.count', 0] }, 0] }, - sdk_versions: { $arrayToObject: '$sdk_versions_array' } + sdks: '$sdk_versions_array' } }; } @@ -153,7 +128,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } } - async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { + async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { const timespanFilter = timeSpan(data.period, data.interval); console.log({ timespanFilter }); const result = await this.db.sdk_report_events @@ -167,7 +142,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.sdkProjectPipeline() ]) .toArray(); - return result[0] as event_types.ListCurrentConnectionsResponse; + return result[0] as event_types.ListCurrentConnections; } async reportSdkConnect(data: event_types.SdkConnectBucketData): Promise { @@ -196,7 +171,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async listCurrentConnections( data: event_types.ListCurrentConnectionsRequest - ): Promise { + ): Promise { const timespanFilter = data.period ? { connect_at: timeSpan(data.period) } : undefined; const result = await this.db.sdk_report_events .aggregate([ @@ -211,7 +186,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.sdkProjectPipeline() ]) .toArray(); - return result[0] as event_types.ListCurrentConnectionsResponse; + return result[0] as event_types.ListCurrentConnections; } async [Symbol.asyncDispose]() { diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorageFactory.ts index bb92dc703..c28db85d6 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorageFactory.ts @@ -3,9 +3,7 @@ import { event_types } from '@powersync/service-types'; export interface ReportStorageFactory extends AsyncDisposable { reportSdkConnect(data: event_types.SdkConnectBucketData): Promise; reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise; - listCurrentConnections( - data: event_types.ListCurrentConnectionsRequest - ): Promise; - scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise; + listCurrentConnections(data: event_types.ListCurrentConnectionsRequest): Promise; + scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise; deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise; } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index a98b53a09..3ce2dce14 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -11,6 +11,12 @@ export type SubscribeEvents = { [EmitterEngineEvents.SDK_DELETE_OLD]: DeleteOldSdkData; }; +export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; +export interface EmitterEvent { + event: K; + handler: EventHandlerFunc; +} + export type SdkUserData = { client_id?: string; user_id: string; @@ -46,22 +52,15 @@ export type SdkConnectDocument = { disconnect_at?: Date; }; -export type InstanceRequest = { - app_id: string; - org_id: string; -}; - export type ListCurrentConnections = { users: number; - // Grouping user_id and sdk versions - user_sdk: number; - // Grouping user_id and client_ids - client_user: number; - // Counts of used sdk versions ( outdated sdks will have the key of unknown) sdks: { - [sdk_version: string]: number; + sdk: string; + users: number; + clients: number; }; }; + export type ScrapeSdkDataRequest = { period: TimeFrames; interval?: number; @@ -70,11 +69,3 @@ export type ScrapeSdkDataRequest = { export type ListCurrentConnectionsRequest = { period?: 'day'; }; - -export type ListCurrentConnectionsResponse = ListCurrentConnections; - -export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; -export interface EmitterEvent { - event: K; - handler: EventHandlerFunc; -} From 70f140872dd2d079babdb0c348e28417303efbcd Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 12:44:22 +0200 Subject: [PATCH 073/169] change to response --- packages/types/src/events.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 3ce2dce14..e80fdd2af 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -58,7 +58,7 @@ export type ListCurrentConnections = { sdk: string; users: number; clients: number; - }; + }[]; }; export type ScrapeSdkDataRequest = { From 5ded1227acb9c438cfd3521a67f87c079cfdb2e8 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 13:05:22 +0200 Subject: [PATCH 074/169] testing date ranges --- .../src/storage/MongoReportStorage.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 0d41bfc34..783bfd574 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -34,15 +34,15 @@ function timeSpan(period: event_types.TimeFrames, timeframe: number = 1): mongo. const { year, month, today } = parseDate(date); switch (period) { case 'month': { - return { $lte: date, $gte: new Date(year, date.getMonth() - timeframe) }; + return { $lte: date, $gt: new Date(year, date.getMonth() - timeframe) }; } case 'week': { const weekStartDate = new Date(date); weekStartDate.setDate(weekStartDate.getDate() - 6 * timeframe); const weekStart = parseDate(weekStartDate); return { - $gte: new Date(weekStart.year, weekStart.month, weekStart.today), - $lte: date + $lte: date, + $gt: new Date(weekStart.year, weekStart.month, weekStart.today) }; } case 'hour': { @@ -57,7 +57,7 @@ function timeSpan(period: event_types.TimeFrames, timeframe: number = 1): mongo. // Start from today to just before tomorrow return { $lte: date, - $gte: new Date(year, month, today - timeframe) + $gt: new Date(year, month, today - timeframe) }; } } From 668b0d81e4c662a1fa6690fb1f8eac18b72bd949 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 14:08:43 +0200 Subject: [PATCH 075/169] custom start range list connections --- .../src/storage/MongoReportStorage.ts | 20 ++++++++++++++----- packages/types/src/events.ts | 5 ++++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 783bfd574..9f470fc01 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -54,7 +54,6 @@ function timeSpan(period: event_types.TimeFrames, timeframe: number = 1): mongo. }; } default: { - // Start from today to just before tomorrow return { $lte: date, $gt: new Date(year, month, today - timeframe) @@ -63,6 +62,20 @@ function timeSpan(period: event_types.TimeFrames, timeframe: number = 1): mongo. } } +function dayDateRange(data: event_types.ListCurrentConnectionsRequest) { + const { range } = data; + if (!range) { + return undefined; + } + const date = new Date(); + return { + connect_at: { + $lte: date, + $gt: new Date(range.start) + } + }; +} + export class MongoReportStorage implements storage.ReportStorageFactory { private readonly client: mongo.MongoClient; public readonly db: PowerSyncMongo; @@ -122,7 +135,6 @@ export class MongoReportStorage implements storage.ReportStorageFactory { connect_at: timeSpan(period, timeframe), $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }] }); - console.log(result); if (result.deletedCount > 0) { logger.info(`TTL: ${result.deletedCount} documents have been removed from sdk_report_events collection`); } @@ -130,7 +142,6 @@ export class MongoReportStorage implements storage.ReportStorageFactory { async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { const timespanFilter = timeSpan(data.period, data.interval); - console.log({ timespanFilter }); const result = await this.db.sdk_report_events .aggregate([ { @@ -172,14 +183,13 @@ export class MongoReportStorage implements storage.ReportStorageFactory { async listCurrentConnections( data: event_types.ListCurrentConnectionsRequest ): Promise { - const timespanFilter = data.period ? { connect_at: timeSpan(data.period) } : undefined; const result = await this.db.sdk_report_events .aggregate([ { $match: { disconnect_at: { $exists: false }, jwt_exp: { $gt: new Date() }, - ...timespanFilter + ...dayDateRange(data) } }, this.sdkFacetPipeline(), diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index e80fdd2af..12f97e0b7 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -67,5 +67,8 @@ export type ScrapeSdkDataRequest = { }; export type ListCurrentConnectionsRequest = { - period?: 'day'; + range?: { + period: 'day'; + start: Date; + }; }; From a2cf616d2b102725dd17e96d020646109557aa51 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 14:18:03 +0200 Subject: [PATCH 076/169] start date --- packages/types/src/events.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 12f97e0b7..73d913071 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -69,6 +69,6 @@ export type ScrapeSdkDataRequest = { export type ListCurrentConnectionsRequest = { range?: { period: 'day'; - start: Date; + start: string; }; }; From 2e6cb414c083d6d7f1e773c7188ddb0f620040ee Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 14:43:36 +0200 Subject: [PATCH 077/169] custom date validation --- .../src/storage/MongoReportStorage.ts | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 9f470fc01..4360f0bba 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -13,49 +13,49 @@ function parseDate(date: Date) { year, month, today, - day + day, + parsedDate: date }; } function updateDocFilter(userId: string, clientId: string) { - const { year, month, today } = parseDate(new Date()); + const { year, month, today, parsedDate } = parseDate(new Date()); const nextDay = today + 1; return { user_id: userId, client_id: clientId, connect_at: { - $gte: new Date(year, month, today), + $gte: parsedDate, $lt: new Date(year, month, nextDay) } }; } function timeSpan(period: event_types.TimeFrames, timeframe: number = 1): mongo.Filter { - const date = new Date(); - const { year, month, today } = parseDate(date); + const { year, month, today, parsedDate } = parseDate(new Date()); switch (period) { case 'month': { - return { $lte: date, $gt: new Date(year, date.getMonth() - timeframe) }; + return { $lte: parsedDate, $gt: new Date(year, parsedDate.getMonth() - timeframe) }; } case 'week': { - const weekStartDate = new Date(date); + const weekStartDate = new Date(parsedDate); weekStartDate.setDate(weekStartDate.getDate() - 6 * timeframe); const weekStart = parseDate(weekStartDate); return { - $lte: date, + $lte: parsedDate, $gt: new Date(weekStart.year, weekStart.month, weekStart.today) }; } case 'hour': { // Get the last hour from the current time - const previousHour = date.getHours() - timeframe; + const previousHour = parsedDate.getHours() - timeframe; return { $gte: new Date(year, month, today, previousHour), - $lt: new Date(year, month, today, date.getHours()) + $lt: new Date(year, month, today, parsedDate.getHours()) }; } default: { return { - $lte: date, + $lte: parsedDate, $gt: new Date(year, month, today - timeframe) }; } @@ -68,9 +68,13 @@ function dayDateRange(data: event_types.ListCurrentConnectionsRequest) { return undefined; } const date = new Date(); + const { today, parsedDate } = parseDate(new Date(range.start)); + if (today - date.getDay() > 2) { + throw new Error('Invalid start date for `day` period. Max period is withing 2 days'); + } return { connect_at: { - $lte: date, + $lte: parsedDate, $gt: new Date(range.start) } }; From c808e97a7a723e865d1b3190af2dcc383b38063e Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 14:46:54 +0200 Subject: [PATCH 078/169] date start validation --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 4 ++-- packages/types/src/events.ts | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 4360f0bba..fe4f538a8 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -68,14 +68,14 @@ function dayDateRange(data: event_types.ListCurrentConnectionsRequest) { return undefined; } const date = new Date(); - const { today, parsedDate } = parseDate(new Date(range.start)); + const { today, parsedDate } = parseDate(new Date(range.start_date)); if (today - date.getDay() > 2) { throw new Error('Invalid start date for `day` period. Max period is withing 2 days'); } return { connect_at: { $lte: parsedDate, - $gt: new Date(range.start) + $gt: new Date(range.start_date) } }; } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 73d913071..b219dce9f 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -68,7 +68,6 @@ export type ScrapeSdkDataRequest = { export type ListCurrentConnectionsRequest = { range?: { - period: 'day'; - start: string; + start_date: string; }; }; From e7e14577d52897316a1035ff5dad0677ee99dfb4 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 14:58:53 +0200 Subject: [PATCH 079/169] testing --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index fe4f538a8..8e5146acd 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -68,8 +68,8 @@ function dayDateRange(data: event_types.ListCurrentConnectionsRequest) { return undefined; } const date = new Date(); - const { today, parsedDate } = parseDate(new Date(range.start_date)); - if (today - date.getDay() > 2) { + const { day, parsedDate } = parseDate(new Date(range.start_date)); + if (day - date.getDay() > 2 || day - date.getDay() < -2) { throw new Error('Invalid start date for `day` period. Max period is withing 2 days'); } return { From 9ccf251d1437450eff1642d73d246396f4f5e26d Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 15:04:22 +0200 Subject: [PATCH 080/169] test --- .../module-mongodb-storage/src/storage/MongoReportStorage.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 8e5146acd..5b4d5477d 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -74,8 +74,8 @@ function dayDateRange(data: event_types.ListCurrentConnectionsRequest) { } return { connect_at: { - $lte: parsedDate, - $gt: new Date(range.start_date) + $lte: date, + $gt: parsedDate } }; } From 0b1faddaa51e038f9f9a1951880b47516b9edcd6 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 16:17:05 +0200 Subject: [PATCH 081/169] refactoring --- .../src/storage/MongoReportStorage.ts | 136 +++++++++--------- .../src/storage/implementation/util.ts | 15 +- .../src/storage/storage-index.ts | 1 + 3 files changed, 84 insertions(+), 68 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 5b4d5477d..4c6c018a2 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -3,6 +3,7 @@ import { storage } from '@powersync/service-core'; import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; import { logger } from '@powersync/lib-services-framework'; +import { parseJsDate } from './implementation/util.js'; function parseDate(date: Date) { const year = date.getFullYear(); @@ -17,68 +18,6 @@ function parseDate(date: Date) { parsedDate: date }; } -function updateDocFilter(userId: string, clientId: string) { - const { year, month, today, parsedDate } = parseDate(new Date()); - const nextDay = today + 1; - return { - user_id: userId, - client_id: clientId, - connect_at: { - $gte: parsedDate, - $lt: new Date(year, month, nextDay) - } - }; -} - -function timeSpan(period: event_types.TimeFrames, timeframe: number = 1): mongo.Filter { - const { year, month, today, parsedDate } = parseDate(new Date()); - switch (period) { - case 'month': { - return { $lte: parsedDate, $gt: new Date(year, parsedDate.getMonth() - timeframe) }; - } - case 'week': { - const weekStartDate = new Date(parsedDate); - weekStartDate.setDate(weekStartDate.getDate() - 6 * timeframe); - const weekStart = parseDate(weekStartDate); - return { - $lte: parsedDate, - $gt: new Date(weekStart.year, weekStart.month, weekStart.today) - }; - } - case 'hour': { - // Get the last hour from the current time - const previousHour = parsedDate.getHours() - timeframe; - return { - $gte: new Date(year, month, today, previousHour), - $lt: new Date(year, month, today, parsedDate.getHours()) - }; - } - default: { - return { - $lte: parsedDate, - $gt: new Date(year, month, today - timeframe) - }; - } - } -} - -function dayDateRange(data: event_types.ListCurrentConnectionsRequest) { - const { range } = data; - if (!range) { - return undefined; - } - const date = new Date(); - const { day, parsedDate } = parseDate(new Date(range.start_date)); - if (day - date.getDay() > 2 || day - date.getDay() < -2) { - throw new Error('Invalid start date for `day` period. Max period is withing 2 days'); - } - return { - connect_at: { - $lte: date, - $gt: parsedDate - } - }; -} export class MongoReportStorage implements storage.ReportStorageFactory { private readonly client: mongo.MongoClient; @@ -133,10 +72,73 @@ export class MongoReportStorage implements storage.ReportStorageFactory { }; } + private updateDocFilter(userId: string, clientId: string) { + const { year, month, today, parsedDate } = parseDate(new Date()); + const nextDay = today + 1; + return { + user_id: userId, + client_id: clientId, + connect_at: { + $gte: parsedDate, + $lt: new Date(year, month, nextDay) + } + }; + } + + private dayDateRange(data: event_types.ListCurrentConnectionsRequest) { + const { range } = data; + if (!range) { + return undefined; + } + const date = new Date(); + const { day, parsedDate } = parseJsDate(new Date(range.start_date)); + if (day - date.getDay() > 2 || day - date.getDay() < -2) { + throw new Error('Invalid start date for `day` period. Max period is withing 2 days'); + } + return { + connect_at: { + $lte: date, + $gt: parsedDate + } + }; + } + + private timeStampQuery(period: event_types.TimeFrames, timeframe: number = 1): mongo.Filter { + const { year, month, today, parsedDate } = parseJsDate(new Date()); + switch (period) { + case 'month': { + return { $lte: parsedDate, $gt: new Date(year, parsedDate.getMonth() - timeframe) }; + } + case 'week': { + const weekStartDate = new Date(parsedDate); + weekStartDate.setDate(weekStartDate.getDate() - 6 * timeframe); + const weekStart = parseJsDate(weekStartDate); + return { + $lte: parsedDate, + $gt: new Date(weekStart.year, weekStart.month, weekStart.today) + }; + } + case 'hour': { + // Get the last hour from the current time + const previousHour = parsedDate.getHours() - timeframe; + return { + $gte: new Date(year, month, today, previousHour), + $lt: new Date(year, month, today, parsedDate.getHours()) + }; + } + default: { + return { + $lte: parsedDate, + $gt: new Date(year, month, today - timeframe) + }; + } + } + } + async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise { const { period, timeframe } = data; const result = await this.db.sdk_report_events.deleteMany({ - connect_at: timeSpan(period, timeframe), + connect_at: this.timeStampQuery(period, timeframe), $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }] }); if (result.deletedCount > 0) { @@ -145,7 +147,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { - const timespanFilter = timeSpan(data.period, data.interval); + const timespanFilter = this.timeStampQuery(data.period, data.interval); const result = await this.db.sdk_report_events .aggregate([ { @@ -162,7 +164,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { async reportSdkConnect(data: event_types.SdkConnectBucketData): Promise { await this.db.sdk_report_events.findOneAndUpdate( - updateDocFilter(data.user_id, data.client_id!), + this.updateDocFilter(data.user_id, data.client_id!), { $set: data, $unset: { @@ -175,7 +177,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { ); } async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { - await this.db.sdk_report_events.findOneAndUpdate(updateDocFilter(data.user_id, data.client_id!), { + await this.db.sdk_report_events.findOneAndUpdate(this.updateDocFilter(data.user_id, data.client_id!), { $set: { disconnect_at: data.disconnect_at }, @@ -193,7 +195,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { $match: { disconnect_at: { $exists: false }, jwt_exp: { $gt: new Date() }, - ...dayDateRange(data) + ...this.dayDateRange(data) } }, this.sdkFacetPipeline(), diff --git a/modules/module-mongodb-storage/src/storage/implementation/util.ts b/modules/module-mongodb-storage/src/storage/implementation/util.ts index 84e94466b..e2488be88 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/util.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/util.ts @@ -1,7 +1,6 @@ import * as bson from 'bson'; import * as crypto from 'crypto'; import * as uuid from 'uuid'; - import { mongo } from '@powersync/lib-service-mongodb'; import { storage, utils } from '@powersync/service-core'; @@ -117,3 +116,17 @@ export const connectMongoForTests = (url: string, isCI: boolean) => { }); return new PowerSyncMongo(client); }; + +export function parseJsDate(date: Date) { + const year = date.getFullYear(); + const month = date.getMonth(); + const today = date.getDate(); + const day = date.getDay(); + return { + year, + month, + today, + day, + parsedDate: date + }; +} diff --git a/modules/module-mongodb-storage/src/storage/storage-index.ts b/modules/module-mongodb-storage/src/storage/storage-index.ts index d4d3373b9..8c934dbc0 100644 --- a/modules/module-mongodb-storage/src/storage/storage-index.ts +++ b/modules/module-mongodb-storage/src/storage/storage-index.ts @@ -12,3 +12,4 @@ export * from './implementation/OperationBatch.js'; export * from './implementation/PersistedBatch.js'; export * from './implementation/util.js'; export * from './MongoBucketStorage.js'; +export * from './MongoReportStorage.js'; From 4ea15e6c7fcf6eb4b863a0a5d8f5ba9520ff9363 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 16:33:03 +0200 Subject: [PATCH 082/169] date bug --- .../src/storage/MongoReportStorage.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 4c6c018a2..98c159b7a 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -163,8 +163,10 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async reportSdkConnect(data: event_types.SdkConnectBucketData): Promise { + const updateFilter = this.updateDocFilter(data.user_id, data.client_id!); + console.log(updateFilter); await this.db.sdk_report_events.findOneAndUpdate( - this.updateDocFilter(data.user_id, data.client_id!), + updateFilter, { $set: data, $unset: { @@ -177,7 +179,9 @@ export class MongoReportStorage implements storage.ReportStorageFactory { ); } async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { - await this.db.sdk_report_events.findOneAndUpdate(this.updateDocFilter(data.user_id, data.client_id!), { + const updateFilter = this.updateDocFilter(data.user_id, data.client_id!); + console.log(updateFilter); + await this.db.sdk_report_events.findOneAndUpdate(updateFilter, { $set: { disconnect_at: data.disconnect_at }, From 3735ab43dc7d407cfecd958f7636a4eb91343224 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 23 Jul 2025 16:39:35 +0200 Subject: [PATCH 083/169] fixes krep in bugs --- .../src/storage/MongoReportStorage.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 98c159b7a..60560d350 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -73,13 +73,14 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } private updateDocFilter(userId: string, clientId: string) { - const { year, month, today, parsedDate } = parseDate(new Date()); + const { year, month, today } = parseDate(new Date()); const nextDay = today + 1; return { user_id: userId, client_id: clientId, connect_at: { - $gte: parsedDate, + // Need to create a new date here to sett the time to 00:00:00 + $gte: new Date(year, month, today), $lt: new Date(year, month, nextDay) } }; @@ -164,7 +165,6 @@ export class MongoReportStorage implements storage.ReportStorageFactory { async reportSdkConnect(data: event_types.SdkConnectBucketData): Promise { const updateFilter = this.updateDocFilter(data.user_id, data.client_id!); - console.log(updateFilter); await this.db.sdk_report_events.findOneAndUpdate( updateFilter, { @@ -180,7 +180,6 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { const updateFilter = this.updateDocFilter(data.user_id, data.client_id!); - console.log(updateFilter); await this.db.sdk_report_events.findOneAndUpdate(updateFilter, { $set: { disconnect_at: data.disconnect_at From c9088a44b7bf9a6bddebf0df7988d74d722b8ea6 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 24 Jul 2025 08:48:11 +0200 Subject: [PATCH 084/169] added seperate filter for delete --- .../src/storage/MongoReportStorage.ts | 55 +++++++++++++++---- packages/types/src/events.ts | 6 +- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 60560d350..6261d3a79 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -104,15 +104,15 @@ export class MongoReportStorage implements storage.ReportStorageFactory { }; } - private timeStampQuery(period: event_types.TimeFrames, timeframe: number = 1): mongo.Filter { + private timeFrameQuery(timeframe: event_types.TimeFrames, interval: number = 1): mongo.Filter { const { year, month, today, parsedDate } = parseJsDate(new Date()); - switch (period) { + switch (timeframe) { case 'month': { - return { $lte: parsedDate, $gt: new Date(year, parsedDate.getMonth() - timeframe) }; + return { $lte: parsedDate, $gt: new Date(year, parsedDate.getMonth() - interval) }; } case 'week': { const weekStartDate = new Date(parsedDate); - weekStartDate.setDate(weekStartDate.getDate() - 6 * timeframe); + weekStartDate.setDate(weekStartDate.getDate() - 6 * interval); const weekStart = parseJsDate(weekStartDate); return { $lte: parsedDate, @@ -121,25 +121,55 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } case 'hour': { // Get the last hour from the current time - const previousHour = parsedDate.getHours() - timeframe; + const previousHour = parsedDate.getHours() - interval; return { - $gte: new Date(year, month, today, previousHour), - $lt: new Date(year, month, today, parsedDate.getHours()) + $gt: new Date(year, month, today, previousHour), + $lte: new Date(year, month, today, parsedDate.getHours()) }; } default: { return { $lte: parsedDate, - $gt: new Date(year, month, today - timeframe) + $gt: new Date(year, month, today - interval) + }; + } + } + } + + private timeFrameDeleteQuery(timeframe: event_types.TimeFrames, interval: number = 1): mongo.Filter { + const { year, month, today, parsedDate } = parseJsDate(new Date()); + switch (timeframe) { + case 'month': { + return { $lt: new Date(year, parsedDate.getMonth() - interval) }; + } + case 'week': { + const weekStartDate = new Date(parsedDate); + weekStartDate.setDate(weekStartDate.getDate() - 6 * interval); + const { month, year, today } = parseJsDate(weekStartDate); + return { + $lt: new Date(year, month, today) + }; + } + case 'hour': { + const previousHour = parsedDate.getHours() - interval; + return { + $lt: new Date(year, month, today, previousHour) + }; + } + default: { + return { + $lt: new Date(year, month, today - interval) }; } } } async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise { - const { period, timeframe } = data; + const { interval, timeframe } = data; + const timeframeFilter = this.timeFrameDeleteQuery(timeframe, interval); + console.log(timeframeFilter); const result = await this.db.sdk_report_events.deleteMany({ - connect_at: this.timeStampQuery(period, timeframe), + connect_at: timeframeFilter, $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }] }); if (result.deletedCount > 0) { @@ -148,12 +178,13 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { - const timespanFilter = this.timeStampQuery(data.period, data.interval); + const { interval, timeframe } = data; + const timeframeFilter = this.timeFrameQuery(timeframe, interval); const result = await this.db.sdk_report_events .aggregate([ { $match: { - connect_at: timespanFilter + connect_at: timeframeFilter } }, this.sdkFacetPipeline(), diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index b219dce9f..16e502b39 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -25,8 +25,8 @@ export type SdkUserData = { }; export type DeleteOldSdkData = { - timeframe: number; - period: TimeFrames; + interval: number; + timeframe: TimeFrames; }; export type SdkConnectEventData = { @@ -62,7 +62,7 @@ export type ListCurrentConnections = { }; export type ScrapeSdkDataRequest = { - period: TimeFrames; + timeframe: TimeFrames; interval?: number; }; From e2f7a7eb61a7f0b75444d50aa1c9a24595db2d11 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 24 Jul 2025 11:10:49 +0200 Subject: [PATCH 085/169] indexes --- .../migrations/1752661449910-sdk-reporting.ts | 12 ++++++--- .../src/storage/MongoReportStorage.ts | 26 ++++++++++++++----- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts index b9d516a9d..ac156951c 100644 --- a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts +++ b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts @@ -17,14 +17,12 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { jwt_exp: 1, disconnect_at: 1 }, - { name: 'connect_at' } + { name: 'list_index' } ); await db.sdk_report_events.createIndex( { - user_id: 1, - sdk: 1, - version: 1 + user_id: 1 }, { name: 'user_id' } ); @@ -34,6 +32,12 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { }, { name: 'client_id' } ); + await db.sdk_report_events.createIndex( + { + sdk: 1 + }, + { name: 'sdk' } + ); } finally { await db.client.close(); } diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 6261d3a79..3fb8ca69a 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -86,7 +86,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { }; } - private dayDateRange(data: event_types.ListCurrentConnectionsRequest) { + private listConnectionsDateRange(data: event_types.ListCurrentConnectionsRequest) { const { range } = data; if (!range) { return undefined; @@ -97,10 +97,8 @@ export class MongoReportStorage implements storage.ReportStorageFactory { throw new Error('Invalid start date for `day` period. Max period is withing 2 days'); } return { - connect_at: { - $lte: date, - $gt: parsedDate - } + $lte: date, + $gt: parsedDate }; } @@ -167,7 +165,6 @@ export class MongoReportStorage implements storage.ReportStorageFactory { async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise { const { interval, timeframe } = data; const timeframeFilter = this.timeFrameDeleteQuery(timeframe, interval); - console.log(timeframeFilter); const result = await this.db.sdk_report_events.deleteMany({ connect_at: timeframeFilter, $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }] @@ -223,13 +220,28 @@ export class MongoReportStorage implements storage.ReportStorageFactory { async listCurrentConnections( data: event_types.ListCurrentConnectionsRequest ): Promise { + const timeframeFilter = this.listConnectionsDateRange(data); + const explain = await this.db.sdk_report_events + .aggregate([ + { + $match: { + disconnect_at: { $exists: false }, + jwt_exp: { $gt: new Date() }, + connect_at: timeframeFilter + } + }, + this.sdkFacetPipeline(), + this.sdkProjectPipeline() + ]) + .explain(); + console.log(explain); const result = await this.db.sdk_report_events .aggregate([ { $match: { disconnect_at: { $exists: false }, jwt_exp: { $gt: new Date() }, - ...this.dayDateRange(data) + connect_at: timeframeFilter } }, this.sdkFacetPipeline(), From 28914bfe7359fbc366b78ff2aced2860dcc439f3 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 24 Jul 2025 11:27:44 +0200 Subject: [PATCH 086/169] check document scan result --- .../src/storage/MongoReportStorage.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 3fb8ca69a..65fa86108 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -178,7 +178,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { const { interval, timeframe } = data; const timeframeFilter = this.timeFrameQuery(timeframe, interval); const result = await this.db.sdk_report_events - .aggregate([ + .aggregate([ { $match: { connect_at: timeframeFilter @@ -188,7 +188,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.sdkProjectPipeline() ]) .toArray(); - return result[0] as event_types.ListCurrentConnections; + return result[0]; } async reportSdkConnect(data: event_types.SdkConnectBucketData): Promise { @@ -234,9 +234,12 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.sdkProjectPipeline() ]) .explain(); - console.log(explain); + explain.stages.map((stage: any) => { + const key = Object.keys(stage)[0]; + console.log(JSON.stringify(stage[key], null, 2)); + }); const result = await this.db.sdk_report_events - .aggregate([ + .aggregate([ { $match: { disconnect_at: { $exists: false }, @@ -248,7 +251,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.sdkProjectPipeline() ]) .toArray(); - return result[0] as event_types.ListCurrentConnections; + return result[0]; } async [Symbol.asyncDispose]() { From 51bbcc47791026ffd10cd9f58cec3ce3ba332521 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 24 Jul 2025 11:34:43 +0200 Subject: [PATCH 087/169] remove explain --- .../src/storage/MongoReportStorage.ts | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 65fa86108..f52893092 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -221,23 +221,6 @@ export class MongoReportStorage implements storage.ReportStorageFactory { data: event_types.ListCurrentConnectionsRequest ): Promise { const timeframeFilter = this.listConnectionsDateRange(data); - const explain = await this.db.sdk_report_events - .aggregate([ - { - $match: { - disconnect_at: { $exists: false }, - jwt_exp: { $gt: new Date() }, - connect_at: timeframeFilter - } - }, - this.sdkFacetPipeline(), - this.sdkProjectPipeline() - ]) - .explain(); - explain.stages.map((stage: any) => { - const key = Object.keys(stage)[0]; - console.log(JSON.stringify(stage[key], null, 2)); - }); const result = await this.db.sdk_report_events .aggregate([ { From db23db6c28386a9eddad9d8852839bafc52dfdcd Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 24 Jul 2025 13:16:06 +0200 Subject: [PATCH 088/169] end date to list connections --- .../src/storage/MongoReportStorage.ts | 11 ++-- .../storage/PostgresReportStorageFactory.ts | 61 +++++++++++++++++++ .../src/storage/PostgresStorageProvider.ts | 8 ++- packages/types/src/events.ts | 1 + 4 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index f52893092..29c2fe65c 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -91,14 +91,11 @@ export class MongoReportStorage implements storage.ReportStorageFactory { if (!range) { return undefined; } - const date = new Date(); - const { day, parsedDate } = parseJsDate(new Date(range.start_date)); - if (day - date.getDay() > 2 || day - date.getDay() < -2) { - throw new Error('Invalid start date for `day` period. Max period is withing 2 days'); - } + const endDate = data.range?.end_date ? new Date(data.range.end_date) : new Date(); + const startDate = new Date(range.start_date); return { - $lte: date, - $gt: parsedDate + $lte: endDate, + $gt: startDate }; } diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts new file mode 100644 index 000000000..4bddde73b --- /dev/null +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -0,0 +1,61 @@ +import { storage } from '@powersync/service-core'; +import * as pg_wire from '@powersync/service-jpgwire'; + +import * as lib_postgres from '@powersync/lib-service-postgres'; +import { NormalizedPostgresStorageConfig } from '../types/types.js'; + +import { STORAGE_SCHEMA_NAME } from '../utils/db.js'; +import { getStorageApplicationName } from '../utils/application-name.js'; +import { + DeleteOldSdkData, + ListCurrentConnections, + ListCurrentConnectionsRequest, + ScrapeSdkDataRequest, + SdkConnectBucketData, + SdkDisconnectEventData +} from '@powersync/service-types/src/events.js'; + +export type PostgresReportStorageOptions = { + config: NormalizedPostgresStorageConfig; +}; + +export class PostgresReportStorageFactory implements storage.ReportStorageFactory { + readonly db: lib_postgres.DatabaseClient; + private activeStorage: storage.SyncRulesBucketStorage | undefined; + + constructor(protected options: PostgresReportStorageOptions) { + this.db = new lib_postgres.DatabaseClient({ + config: options.config, + schema: STORAGE_SCHEMA_NAME, + applicationName: getStorageApplicationName() + }); + + this.db.registerListener({ + connectionCreated: async (connection) => this.prepareStatements(connection) + }); + } + reportSdkConnect(data: SdkConnectBucketData): Promise { + throw new Error('Method not implemented.'); + } + reportSdkDisconnect(data: SdkDisconnectEventData): Promise { + throw new Error('Method not implemented.'); + } + listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { + throw new Error('Method not implemented.'); + } + scrapeSdkData(data: ScrapeSdkDataRequest): Promise { + throw new Error('Method not implemented.'); + } + deleteOldSdkData(data: DeleteOldSdkData): Promise { + throw new Error('Method not implemented.'); + } + + async [Symbol.asyncDispose]() { + await this.db[Symbol.asyncDispose](); + } + + async prepareStatements(connection: pg_wire.PgConnection) { + // It should be possible to prepare statements for some common operations here. + // This has not been implemented yet. + } +} diff --git a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts index 12e25c184..c831d4023 100644 --- a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts +++ b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts @@ -5,6 +5,7 @@ import { storage } from '@powersync/service-core'; import { isPostgresStorageConfig, normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js'; import { dropTables } from '../utils/db.js'; import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; +import { PostgresReportStorageFactory } from './PostgresReportStorageFactory.js'; export class PostgresStorageProvider implements storage.BucketStorageProvider { get type() { @@ -28,9 +29,14 @@ export class PostgresStorageProvider implements storage.BucketStorageProvider { config: normalizedConfig, slot_name_prefix: options.resolvedConfig.slot_name_prefix }); + + const reportStorageFactory = new PostgresReportStorageFactory({ + config: normalizedConfig + }); + return { // TODO: IMPLEMENT REPORT STORAGE - reportStorage: null, + reportStorage: reportStorageFactory, storage: storageFactory, shutDown: async () => storageFactory.db[Symbol.asyncDispose](), tearDown: async () => { diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 16e502b39..f5700f5ce 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -69,5 +69,6 @@ export type ScrapeSdkDataRequest = { export type ListCurrentConnectionsRequest = { range?: { start_date: string; + end_date?: string; }; }; From f67746d431062c9fb115bf9e919b323602a4b33d Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 24 Jul 2025 13:46:27 +0200 Subject: [PATCH 089/169] removed a log --- packages/service-core/src/emitters/EmitterEngine.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index c7e21bf25..ff1932452 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -29,9 +29,6 @@ export class EmitterEngine implements BaseEmitterEngine { } emit(event: K, data: event_types.SubscribeEvents[K]): void { - if (!this.events.has(event) || this.countListeners(event) === 0) { - logger.warn(`${event} has no listener registered.`); - } this.emitter.emit(event, data); } From 9797bad1c87ac236b12932e15320bb6786d84812 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 12:14:29 +0200 Subject: [PATCH 090/169] postgres imp WIP --- .../migrations/1752661449910-sdk-reporting.ts | 8 +- .../src/storage/MongoReportStorage.ts | 39 ++- .../src/storage/implementation/util.ts | 14 - .../migrations/scripts/1684951997326-init.ts | 22 ++ .../storage/PostgresReportStorageFactory.ts | 258 +++++++++++++++++- .../src/storage/PostgresStorageProvider.ts | 7 +- 6 files changed, 295 insertions(+), 53 deletions(-) diff --git a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts index ac156951c..9ad09e9df 100644 --- a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts +++ b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts @@ -17,26 +17,26 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { jwt_exp: 1, disconnect_at: 1 }, - { name: 'list_index' } + { name: 'sdk_list_index' } ); await db.sdk_report_events.createIndex( { user_id: 1 }, - { name: 'user_id' } + { name: 'sdk_user_id_index' } ); await db.sdk_report_events.createIndex( { client_id: 1 }, - { name: 'client_id' } + { name: 'sdk_client_id_index' } ); await db.sdk_report_events.createIndex( { sdk: 1 }, - { name: 'sdk' } + { name: 'sdk_index' } ); } finally { await db.client.close(); diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 29c2fe65c..3d75db0dc 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -3,21 +3,6 @@ import { storage } from '@powersync/service-core'; import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; import { logger } from '@powersync/lib-services-framework'; -import { parseJsDate } from './implementation/util.js'; - -function parseDate(date: Date) { - const year = date.getFullYear(); - const month = date.getMonth(); - const today = date.getDate(); - const day = date.getDay(); - return { - year, - month, - today, - day, - parsedDate: date - }; -} export class MongoReportStorage implements storage.ReportStorageFactory { private readonly client: mongo.MongoClient; @@ -28,6 +13,20 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.db = db; } + private parseJsDate(date: Date) { + const year = date.getFullYear(); + const month = date.getMonth(); + const today = date.getDate(); + const day = date.getDay(); + return { + year, + month, + today, + day, + parsedDate: date + }; + } + private sdkFacetPipeline() { return { $facet: { @@ -73,7 +72,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } private updateDocFilter(userId: string, clientId: string) { - const { year, month, today } = parseDate(new Date()); + const { year, month, today } = this.parseJsDate(new Date()); const nextDay = today + 1; return { user_id: userId, @@ -100,7 +99,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } private timeFrameQuery(timeframe: event_types.TimeFrames, interval: number = 1): mongo.Filter { - const { year, month, today, parsedDate } = parseJsDate(new Date()); + const { year, month, today, parsedDate } = this.parseJsDate(new Date()); switch (timeframe) { case 'month': { return { $lte: parsedDate, $gt: new Date(year, parsedDate.getMonth() - interval) }; @@ -108,7 +107,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { case 'week': { const weekStartDate = new Date(parsedDate); weekStartDate.setDate(weekStartDate.getDate() - 6 * interval); - const weekStart = parseJsDate(weekStartDate); + const weekStart = this.parseJsDate(weekStartDate); return { $lte: parsedDate, $gt: new Date(weekStart.year, weekStart.month, weekStart.today) @@ -132,7 +131,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } private timeFrameDeleteQuery(timeframe: event_types.TimeFrames, interval: number = 1): mongo.Filter { - const { year, month, today, parsedDate } = parseJsDate(new Date()); + const { year, month, today, parsedDate } = this.parseJsDate(new Date()); switch (timeframe) { case 'month': { return { $lt: new Date(year, parsedDate.getMonth() - interval) }; @@ -140,7 +139,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { case 'week': { const weekStartDate = new Date(parsedDate); weekStartDate.setDate(weekStartDate.getDate() - 6 * interval); - const { month, year, today } = parseJsDate(weekStartDate); + const { month, year, today } = this.parseJsDate(weekStartDate); return { $lt: new Date(year, month, today) }; diff --git a/modules/module-mongodb-storage/src/storage/implementation/util.ts b/modules/module-mongodb-storage/src/storage/implementation/util.ts index e2488be88..b0b786375 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/util.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/util.ts @@ -116,17 +116,3 @@ export const connectMongoForTests = (url: string, isCI: boolean) => { }); return new PowerSyncMongo(client); }; - -export function parseJsDate(date: Date) { - const year = date.getFullYear(); - const month = date.getMonth(); - const today = date.getDate(); - const day = date.getDay(); - return { - year, - month, - today, - day, - parsedDate: date - }; -} diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index 61e8699ea..dc7f6d2ae 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -128,6 +128,28 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { CONSTRAINT unique_user_sync PRIMARY KEY (user_id, sync_rules_id) ); `.execute(); + + await db.sql` + CREATE TABLE sdk_report_events ( + id TEXT PRIMARY KEY, + user_agent TEXT NOT NULL, + client_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sdk TEXT NOT NULL, + jwt_exp TIMESTAMP WITH TIME ZONE, + connect_at TIMESTAMP WITH TIME ZONE NOT NULL, + disconnect_at TIMESTAMP WITH TIME ZONE, + CONSTRAINT unique_user_client_connect UNIQUE (user_id, client_id, connect_at) + ) + `.execute(); + + await db.sql` CREATE INDEX sdk_list_index ON sdk_report_events (connect_at, jwt_exp, disconnect_at) `.execute(); + + await db.sql` CREATE INDEX sdk_user_id_index ON sdk_report_events (user_id)`.execute(); + + await db.sql` CREATE INDEX sdk_client_id_index ON sdk_report_events (client_id)`.execute(); + + await db.sql` CREATE INDEX sdk_index ON sdk_report_events (sdk)`.execute(); }); }; diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 4bddde73b..3e1bc8375 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -1,6 +1,7 @@ import { storage } from '@powersync/service-core'; import * as pg_wire from '@powersync/service-jpgwire'; - +import { event_types } from '@powersync/service-types'; +import { v4 } from 'uuid'; import * as lib_postgres from '@powersync/lib-service-postgres'; import { NormalizedPostgresStorageConfig } from '../types/types.js'; @@ -21,8 +22,6 @@ export type PostgresReportStorageOptions = { export class PostgresReportStorageFactory implements storage.ReportStorageFactory { readonly db: lib_postgres.DatabaseClient; - private activeStorage: storage.SyncRulesBucketStorage | undefined; - constructor(protected options: PostgresReportStorageOptions) { this.db = new lib_postgres.DatabaseClient({ config: options.config, @@ -34,20 +33,253 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor connectionCreated: async (connection) => this.prepareStatements(connection) }); } - reportSdkConnect(data: SdkConnectBucketData): Promise { - throw new Error('Method not implemented.'); + + private parseJsDate(date: Date) { + const year = date.getFullYear(); + const month = date.getMonth(); + const today = date.getDate(); + const day = date.getDay(); + return { + year, + month, + today, + day, + parsedDate: date + }; } - reportSdkDisconnect(data: SdkDisconnectEventData): Promise { - throw new Error('Method not implemented.'); + private timeFrameQuery(timeframe: event_types.TimeFrames, interval: number = 1) { + const { year, month, today, parsedDate } = this.parseJsDate(new Date()); + switch (timeframe) { + case 'month': { + return { lt: parsedDate.toISOString(), gt: new Date(year, parsedDate.getMonth() - interval).toISOString() }; + } + case 'week': { + const weekStartDate = new Date(parsedDate); + weekStartDate.setDate(weekStartDate.getDate() - 6 * interval); + const weekStart = this.parseJsDate(weekStartDate); + return { + lt: parsedDate.toISOString(), + gt: new Date(weekStart.year, weekStart.month, weekStart.today).toISOString() + }; + } + case 'hour': { + // Get the last hour from the current time + const previousHour = parsedDate.getHours() - interval; + return { + lt: new Date(year, month, today, parsedDate.getHours()).toISOString(), + gt: new Date(year, month, today, previousHour).toISOString() + }; + } + default: { + return { + lt: parsedDate.toISOString(), + gt: new Date(year, month, today - interval).toISOString() + }; + } + } } - listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { - throw new Error('Method not implemented.'); + + private timeFrameDeleteQuery(timeframe: event_types.TimeFrames, interval: number = 1) { + const { year, month, today, parsedDate } = this.parseJsDate(new Date()); + switch (timeframe) { + case 'month': { + return { lt: new Date(year, parsedDate.getMonth() - interval).toISOString() }; + } + case 'week': { + const weekStartDate = new Date(parsedDate); + weekStartDate.setDate(weekStartDate.getDate() - 6 * interval); + const { month, year, today } = this.parseJsDate(weekStartDate); + return { + lt: new Date(year, month, today).toISOString() + }; + } + case 'hour': { + const previousHour = parsedDate.getHours() - interval; + return { + lt: new Date(year, month, today, previousHour).toISOString() + }; + } + default: { + return { + $lt: new Date(year, month, today - interval).toISOString() + }; + } + } } - scrapeSdkData(data: ScrapeSdkDataRequest): Promise { - throw new Error('Method not implemented.'); + + private listConnectionsDateRangeQuery(data: event_types.ListCurrentConnectionsRequest) { + const { range } = data; + if (!range) { + const query = ` + WITH filtered AS ( + SELECT * + FROM sdk_report_events + WHERE disconnect_at IS NULL + AND jwt_exp > NOW() + ), + unique_users AS ( + SELECT COUNT(DISTINCT user_id) AS count + FROM filtered + ), + sdk_versions_array AS ( + SELECT sdk, + COUNT(*) AS total, + COUNT(DISTINCT client_id) AS clients, + COUNT(DISTINCT user_id) AS users + FROM filtered + GROUP BY sdk + ) + SELECT + COALESCE(u.count, 0) AS users, + JSON_AGG(ROW_TO_JSON(s)) AS sdks + FROM unique_users u + JOIN sdk_versions_array s ON TRUE; + `; + return { + statement: query + }; + } + const endDate = data.range?.end_date ? new Date(data.range.end_date) : new Date(); + const startDate = new Date(range.start_date); + const query = ` + WITH filtered AS ( + SELECT * + FROM sdk_report_events + WHERE disconnect_at IS NULL + AND jwt_exp > NOW() + AND connect_at > $1 + AND connect_at <= $2 + ), + unique_users AS ( + SELECT COUNT(DISTINCT user_id) AS count + FROM filtered + ), + sdk_versions_array AS ( + SELECT sdk, + COUNT(*) AS total, + COUNT(DISTINCT client_id) AS clients, + COUNT(DISTINCT user_id) AS users + FROM filtered + GROUP BY sdk + ) + SELECT COALESCE(u.count, 0) AS users, JSON_AGG(ROW_TO_JSON(s)) AS sdks + FROM unique_users u + JOIN sdk_versions_array s ON TRUE; + `; + const lt = endDate.toISOString(); + const gt = startDate.toISOString(); + return { statement: query, params: [{ value: gt }, { value: lt }] }; + } + + private updateTableFilter() { + const { year, month, today } = this.parseJsDate(new Date()); + const nextDay = today + 1; + return { + gte: new Date(year, month, today).toISOString(), + lt: new Date(year, month, nextDay).toISOString() + }; + } + + async reportSdkConnect(data: SdkConnectBucketData): Promise { + const { sdk, connect_at, user_id, user_agent, jwt_exp, client_id } = data; + const { gte, lt } = this.updateTableFilter(); + const query = ` + INSERT INTO sdk_report_events (user_id, client_id, connect_at, sdk, user_agent, jwt_exp, id) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (user_id, client_id, connect_at) + DO UPDATE SET + connect_at = $3, + sdk = $4, + user_agent = $5, + jwt_exp = $6, + disconnect_at = NULL + WHERE sdk_report_events.connect_at >= $8 + AND sdk_report_events.connect_at < $9;`; + const params = [ + { value: user_id }, + { value: client_id }, + { value: connect_at }, + { value: sdk }, + { value: user_agent }, + { value: jwt_exp }, + { value: v4() }, + { value: gte }, + { value: lt } + ]; + await this.db.query({ statement: query, params }); + } + async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { + const { user_id, client_id, disconnect_at } = data; + const { gte, lt } = this.updateTableFilter(); + const query = ` + UPDATE sdk_report_events + SET + disconnect_at = $1, + jwt_exp = NULL + WHERE user_id = $2 + AND client_id = $3 + AND connect_at >= $4 + AND connect_at < $5;`; + const params = [{ value: disconnect_at }, { value: user_id }, { value: client_id }, { value: gte }, { value: lt }]; + await this.db.query({ statement: query, params }); + } + async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { + const statement = this.listConnectionsDateRangeQuery(data); + const result = await this.db.query(statement); + console.log(result.rows); + return { + users: 0, + sdks: [] + }; + } + + async scrapeSdkData(data: ScrapeSdkDataRequest): Promise { + const { timeframe, interval } = data; + const { lt, gt } = this.timeFrameQuery(timeframe, interval); + const query = ` + WITH filtered AS ( + SELECT * + FROM sdk_report_events + WHERE connect_at > $1 + AND connect_at <= $2 + ), + unique_users AS ( + SELECT COUNT(DISTINCT user_id) AS count + FROM filtered + ), + sdk_versions_array AS ( + SELECT sdk, + COUNT(*) AS total, + COUNT(DISTINCT client_id) AS clients, + COUNT(DISTINCT user_id) AS users + FROM filtered + GROUP BY sdk + ) + SELECT COALESCE(u.count, 0) AS users, JSON_AGG(ROW_TO_JSON(s)) AS sdks + FROM unique_users u + JOIN sdk_versions_array s ON TRUE; + `; + const result = await this.db.query({ statement: query, params: [{ value: gt }, { value: lt }] }); + console.log(result.rows); + return { + users: 0, + sdks: [] + }; } - deleteOldSdkData(data: DeleteOldSdkData): Promise { - throw new Error('Method not implemented.'); + async deleteOldSdkData(data: DeleteOldSdkData): Promise { + const { timeframe, interval } = data; + const { lt } = this.timeFrameDeleteQuery(timeframe, interval); + const query = ` + DELETE FROM sdk_report_events + WHERE connect_at < $1 + AND ( + disconnect_at IS NOT NULL + OR (jwt_exp < NOW() AND disconnect_at IS NULL) + ); +`; + const params = [{ value: lt }]; + const result = await this.db.query({ statement: query, params }); + console.log(result.rows); } async [Symbol.asyncDispose]() { diff --git a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts index c831d4023..749692ff4 100644 --- a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts +++ b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts @@ -35,14 +35,17 @@ export class PostgresStorageProvider implements storage.BucketStorageProvider { }); return { - // TODO: IMPLEMENT REPORT STORAGE reportStorage: reportStorageFactory, storage: storageFactory, - shutDown: async () => storageFactory.db[Symbol.asyncDispose](), + shutDown: async () => { + await storageFactory.db[Symbol.asyncDispose](); + await reportStorageFactory.db[Symbol.asyncDispose](); + }, tearDown: async () => { logger.info(`Tearing down Postgres storage: ${normalizedConfig.database}...`); await dropTables(storageFactory.db); await storageFactory.db[Symbol.asyncDispose](); + await reportStorageFactory.db[Symbol.asyncDispose](); return true; } } satisfies storage.ActiveStorage; From 4682a74959a2bca417d82bd0d304fe471541d208 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 14:13:02 +0200 Subject: [PATCH 091/169] tests --- .../storage/PostgresReportStorageFactory.ts | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 3e1bc8375..96c362038 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -195,18 +195,25 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor disconnect_at = NULL WHERE sdk_report_events.connect_at >= $8 AND sdk_report_events.connect_at < $9;`; - const params = [ - { value: user_id }, - { value: client_id }, - { value: connect_at }, - { value: sdk }, - { value: user_agent }, - { value: jwt_exp }, - { value: v4() }, - { value: gte }, - { value: lt } - ]; - await this.db.query({ statement: query, params }); + try { + const result = await this.db.query({ + statement: query, + params: [ + { type: 'varchar', value: user_id }, + { type: 'varchar', value: client_id }, + { type: 'varchar', value: connect_at }, + { type: 'varchar', value: sdk }, + { type: 'varchar', value: user_agent }, + { type: 1184, value: jwt_exp?.toISOString() }, + { type: 'varchar', value: v4() }, + { type: 1184, value: gte }, + { type: 1184, value: lt } + ] + }); + console.log(result.rows); + } catch (error) { + console.log(error); + } } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { const { user_id, client_id, disconnect_at } = data; @@ -220,8 +227,22 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor AND client_id = $3 AND connect_at >= $4 AND connect_at < $5;`; - const params = [{ value: disconnect_at }, { value: user_id }, { value: client_id }, { value: gte }, { value: lt }]; - await this.db.query({ statement: query, params }); + + try { + const result = await this.db.query({ + statement: query, + params: [ + { type: 1184, value: disconnect_at }, + { type: 'varchar', value: user_id }, + { type: 'varchar', value: client_id }, + { type: 1184, value: gte }, + { type: 1184, value: lt } + ] + }); + console.log(result.rows); + } catch (error) { + console.log(error); + } } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { const statement = this.listConnectionsDateRangeQuery(data); From f46bede6af6642ed505a0a90bd714a73e53d6241 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 14:46:07 +0200 Subject: [PATCH 092/169] timestamp casting --- .../storage/PostgresReportStorageFactory.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 96c362038..c1a1a8d59 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -147,8 +147,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor FROM sdk_report_events WHERE disconnect_at IS NULL AND jwt_exp > NOW() - AND connect_at > $1 - AND connect_at <= $2 + AND connect_at > TIMESTAMP WITH TIME ZONE $1 + AND connect_at <= TIMESTAMP WITH TIME ZONE $2 ), unique_users AS ( SELECT COUNT(DISTINCT user_id) AS count @@ -188,13 +188,13 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (user_id, client_id, connect_at) DO UPDATE SET - connect_at = $3, + connect_at = TIMESTAMP WITH TIME ZONE $3, sdk = $4, user_agent = $5, - jwt_exp = $6, + jwt_exp = TIMESTAMP WITH TIME ZONE $6, disconnect_at = NULL - WHERE sdk_report_events.connect_at >= $8 - AND sdk_report_events.connect_at < $9;`; + WHERE sdk_report_events.connect_at >= TIMESTAMP WITH TIME ZONE $8 + AND sdk_report_events.connect_at < TIMESTAMP WITH TIME ZONE $9;`; try { const result = await this.db.query({ statement: query, @@ -225,8 +225,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor jwt_exp = NULL WHERE user_id = $2 AND client_id = $3 - AND connect_at >= $4 - AND connect_at < $5;`; + AND connect_at >= TIMESTAMP WITH TIME ZONE $4 + AND connect_at < TIMESTAMP WITH TIME ZONE $5;`; try { const result = await this.db.query({ @@ -261,8 +261,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor WITH filtered AS ( SELECT * FROM sdk_report_events - WHERE connect_at > $1 - AND connect_at <= $2 + WHERE connect_at > TIMESTAMP WITH TIME ZONE $1 + AND connect_at <= TIMESTAMP WITH TIME ZONE $2 ), unique_users AS ( SELECT COUNT(DISTINCT user_id) AS count @@ -292,7 +292,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const { lt } = this.timeFrameDeleteQuery(timeframe, interval); const query = ` DELETE FROM sdk_report_events - WHERE connect_at < $1 + WHERE connect_at < TIMESTAMP WITH TIME ZONE $1 AND ( disconnect_at IS NOT NULL OR (jwt_exp < NOW() AND disconnect_at IS NULL) From 8d194f1b02dd8e418b8f0ff512734c2a32dcb266 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 15:02:16 +0200 Subject: [PATCH 093/169] casting issues --- .../storage/PostgresReportStorageFactory.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index c1a1a8d59..1fa20387c 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -147,8 +147,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor FROM sdk_report_events WHERE disconnect_at IS NULL AND jwt_exp > NOW() - AND connect_at > TIMESTAMP WITH TIME ZONE $1 - AND connect_at <= TIMESTAMP WITH TIME ZONE $2 + AND connect_at > CAST($1 AS TIMESTAMP WITH TIME ZONE) + AND connect_at <= CAST($2 AS TIMESTAMP WITH TIME ZONE) ), unique_users AS ( SELECT COUNT(DISTINCT user_id) AS count @@ -188,13 +188,13 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (user_id, client_id, connect_at) DO UPDATE SET - connect_at = TIMESTAMP WITH TIME ZONE $3, + connect_at = CAST($3 AS TIMESTAMP WITH TIME ZONE), sdk = $4, user_agent = $5, - jwt_exp = TIMESTAMP WITH TIME ZONE $6, + jwt_exp = CAST($6 AS TIMESTAMP WITH TIME ZONE), disconnect_at = NULL - WHERE sdk_report_events.connect_at >= TIMESTAMP WITH TIME ZONE $8 - AND sdk_report_events.connect_at < TIMESTAMP WITH TIME ZONE $9;`; + WHERE sdk_report_events.connect_at >= CAST($8 AS TIMESTAMP WITH TIME ZONE) + AND sdk_report_events.connect_at < CAST($9 AS TIMESTAMP WITH TIME ZONE);`; try { const result = await this.db.query({ statement: query, @@ -221,12 +221,12 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const query = ` UPDATE sdk_report_events SET - disconnect_at = $1, + disconnect_at = CAST($1 AS TIMESTAMP WITH TIME ZONE), jwt_exp = NULL WHERE user_id = $2 AND client_id = $3 - AND connect_at >= TIMESTAMP WITH TIME ZONE $4 - AND connect_at < TIMESTAMP WITH TIME ZONE $5;`; + AND connect_at >= CAST($4 AS TIMESTAMP WITH TIME ZONE) + AND connect_at < CAST($5 AS TIMESTAMP WITH TIME ZONE);`; try { const result = await this.db.query({ @@ -261,8 +261,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor WITH filtered AS ( SELECT * FROM sdk_report_events - WHERE connect_at > TIMESTAMP WITH TIME ZONE $1 - AND connect_at <= TIMESTAMP WITH TIME ZONE $2 + WHERE connect_at > CAST($1 AS TIMESTAMP WITH TIME ZONE) + AND connect_at <= CAST($2 AS TIMESTAMP WITH TIME ZONE) ), unique_users AS ( SELECT COUNT(DISTINCT user_id) AS count @@ -292,7 +292,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const { lt } = this.timeFrameDeleteQuery(timeframe, interval); const query = ` DELETE FROM sdk_report_events - WHERE connect_at < TIMESTAMP WITH TIME ZONE $1 + WHERE connect_at < CAST($1 AS TIMESTAMP WITH TIME ZONE) AND ( disconnect_at IS NOT NULL OR (jwt_exp < NOW() AND disconnect_at IS NULL) From 29b4b59ae1086172e2e210d36326812699815412 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 15:10:43 +0200 Subject: [PATCH 094/169] fix incorrect type --- .../src/storage/PostgresReportStorageFactory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 1fa20387c..c4d120847 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -201,7 +201,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor params: [ { type: 'varchar', value: user_id }, { type: 'varchar', value: client_id }, - { type: 'varchar', value: connect_at }, + { type: 1184, value: connect_at }, { type: 'varchar', value: sdk }, { type: 'varchar', value: user_agent }, { type: 1184, value: jwt_exp?.toISOString() }, From a741f417689a253598663d3f6fb1630c10e4a4bf Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 15:26:01 +0200 Subject: [PATCH 095/169] iso issues --- .../storage/PostgresReportStorageFactory.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index c4d120847..33dce34c1 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -49,16 +49,17 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } private timeFrameQuery(timeframe: event_types.TimeFrames, interval: number = 1) { const { year, month, today, parsedDate } = this.parseJsDate(new Date()); + const parsedIsoString = parsedDate.toISOString(); switch (timeframe) { case 'month': { - return { lt: parsedDate.toISOString(), gt: new Date(year, parsedDate.getMonth() - interval).toISOString() }; + return { lt: parsedIsoString, gt: new Date(year, parsedDate.getMonth() - interval).toISOString() }; } case 'week': { const weekStartDate = new Date(parsedDate); weekStartDate.setDate(weekStartDate.getDate() - 6 * interval); const weekStart = this.parseJsDate(weekStartDate); return { - lt: parsedDate.toISOString(), + lt: parsedIsoString, gt: new Date(weekStart.year, weekStart.month, weekStart.today).toISOString() }; } @@ -72,7 +73,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } default: { return { - lt: parsedDate.toISOString(), + lt: parsedIsoString, gt: new Date(year, month, today - interval).toISOString() }; } @@ -101,7 +102,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } default: { return { - $lt: new Date(year, month, today - interval).toISOString() + lt: new Date(year, month, today - interval).toISOString() }; } } @@ -182,6 +183,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor async reportSdkConnect(data: SdkConnectBucketData): Promise { const { sdk, connect_at, user_id, user_agent, jwt_exp, client_id } = data; + const connectIsoString = connect_at.toISOString(); + const jwtExpIsoString = jwt_exp!.toISOString(); const { gte, lt } = this.updateTableFilter(); const query = ` INSERT INTO sdk_report_events (user_id, client_id, connect_at, sdk, user_agent, jwt_exp, id) @@ -201,10 +204,10 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor params: [ { type: 'varchar', value: user_id }, { type: 'varchar', value: client_id }, - { type: 1184, value: connect_at }, + { type: 1184, value: connectIsoString }, { type: 'varchar', value: sdk }, { type: 'varchar', value: user_agent }, - { type: 1184, value: jwt_exp?.toISOString() }, + { type: 1184, value: jwtExpIsoString }, { type: 'varchar', value: v4() }, { type: 1184, value: gte }, { type: 1184, value: lt } @@ -217,6 +220,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { const { user_id, client_id, disconnect_at } = data; + const disconnectIsoString = disconnect_at.toISOString(); const { gte, lt } = this.updateTableFilter(); const query = ` UPDATE sdk_report_events @@ -232,7 +236,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const result = await this.db.query({ statement: query, params: [ - { type: 1184, value: disconnect_at }, + { type: 1184, value: disconnectIsoString }, { type: 'varchar', value: user_id }, { type: 'varchar', value: client_id }, { type: 1184, value: gte }, From 75c66921bfe05bc1c54621a067d47228eefd374b Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 15:55:40 +0200 Subject: [PATCH 096/169] testing --- .../storage/PostgresReportStorageFactory.ts | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 33dce34c1..df42c4b3a 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -186,20 +186,36 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const connectIsoString = connect_at.toISOString(); const jwtExpIsoString = jwt_exp!.toISOString(); const { gte, lt } = this.updateTableFilter(); - const query = ` - INSERT INTO sdk_report_events (user_id, client_id, connect_at, sdk, user_agent, jwt_exp, id) - VALUES ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (user_id, client_id, connect_at) - DO UPDATE SET - connect_at = CAST($3 AS TIMESTAMP WITH TIME ZONE), - sdk = $4, - user_agent = $5, - jwt_exp = CAST($6 AS TIMESTAMP WITH TIME ZONE), - disconnect_at = NULL - WHERE sdk_report_events.connect_at >= CAST($8 AS TIMESTAMP WITH TIME ZONE) - AND sdk_report_events.connect_at < CAST($9 AS TIMESTAMP WITH TIME ZONE);`; try { - const result = await this.db.query({ + await this.db.query('BEGIN'); + const query = ` + UPDATE sdk_report_events + SET + connect_at = $3::timestamptz, + sdk = $4, + user_agent = $5, + jwt_exp = $6::timestamptz, + disconnect_at = NULL + WHERE user_id = $1 + AND client_id = $2 + AND connect_at = $3::timestamptz + AND connect_at >= $8::timestamptz + AND connect_at < $9::timestamptz; + + INSERT INTO sdk_report_events ( + user_id, client_id, connect_at, sdk, user_agent, jwt_exp, id + ) + SELECT $1, $2, $3::timestamptz, $4, $5, $6::timestamptz, $7 + WHERE NOT EXISTS ( + SELECT 1 FROM sdk_report_events + WHERE user_id = $1 + AND client_id = $2 + AND connect_at = $3::timestamptz + AND connect_at >= $8::timestamptz + AND connect_at < $9::timestamptz + );`; + + await this.db.query({ statement: query, params: [ { type: 'varchar', value: user_id }, @@ -213,9 +229,10 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor { type: 1184, value: lt } ] }); - console.log(result.rows); + await this.db.query('COMMIT'); } catch (error) { console.log(error); + await this.db.query('ROLLBACK'); } } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { From 70eac9c83ea441ac055911a7e456621e2910506e Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 15:58:41 +0200 Subject: [PATCH 097/169] testing --- .../src/storage/PostgresReportStorageFactory.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index df42c4b3a..4fc94f562 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -187,8 +187,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const jwtExpIsoString = jwt_exp!.toISOString(); const { gte, lt } = this.updateTableFilter(); try { - await this.db.query('BEGIN'); const query = ` + BEGIN; UPDATE sdk_report_events SET connect_at = $3::timestamptz, @@ -213,7 +213,9 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor AND connect_at = $3::timestamptz AND connect_at >= $8::timestamptz AND connect_at < $9::timestamptz - );`; + ); + + COMMIT;`; await this.db.query({ statement: query, @@ -229,7 +231,6 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor { type: 1184, value: lt } ] }); - await this.db.query('COMMIT'); } catch (error) { console.log(error); await this.db.query('ROLLBACK'); From 732e1d2947ae3e6a95fee0c3d38659c38670591e Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 16:13:43 +0200 Subject: [PATCH 098/169] transaction --- .../src/storage/PostgresReportStorageFactory.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 4fc94f562..72d44e989 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -187,8 +187,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const jwtExpIsoString = jwt_exp!.toISOString(); const { gte, lt } = this.updateTableFilter(); try { + await this.db.query('BEGIN'); const query = ` - BEGIN; UPDATE sdk_report_events SET connect_at = $3::timestamptz, @@ -198,7 +198,6 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor disconnect_at = NULL WHERE user_id = $1 AND client_id = $2 - AND connect_at = $3::timestamptz AND connect_at >= $8::timestamptz AND connect_at < $9::timestamptz; @@ -210,12 +209,9 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor SELECT 1 FROM sdk_report_events WHERE user_id = $1 AND client_id = $2 - AND connect_at = $3::timestamptz AND connect_at >= $8::timestamptz AND connect_at < $9::timestamptz - ); - - COMMIT;`; + );`; await this.db.query({ statement: query, @@ -231,6 +227,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor { type: 1184, value: lt } ] }); + await this.db.query('COMMIT'); } catch (error) { console.log(error); await this.db.query('ROLLBACK'); From 8899f9eed15ea1cc9a7876d1874ae8b6c49b8287 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 16:20:11 +0200 Subject: [PATCH 099/169] added logs --- .../storage/PostgresReportStorageFactory.ts | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 72d44e989..47ffd5978 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -187,15 +187,13 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const jwtExpIsoString = jwt_exp!.toISOString(); const { gte, lt } = this.updateTableFilter(); try { - await this.db.query('BEGIN'); const query = ` UPDATE sdk_report_events - SET - connect_at = $3::timestamptz, - sdk = $4, - user_agent = $5, - jwt_exp = $6::timestamptz, - disconnect_at = NULL + SET connect_at = $3::timestamptz, + sdk = $4, + user_agent = $5, + jwt_exp = $6::timestamptz, + disconnect_at = NULL WHERE user_id = $1 AND client_id = $2 AND connect_at >= $8::timestamptz @@ -203,14 +201,14 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor INSERT INTO sdk_report_events ( user_id, client_id, connect_at, sdk, user_agent, jwt_exp, id - ) + ) SELECT $1, $2, $3::timestamptz, $4, $5, $6::timestamptz, $7 WHERE NOT EXISTS ( SELECT 1 FROM sdk_report_events WHERE user_id = $1 - AND client_id = $2 - AND connect_at >= $8::timestamptz - AND connect_at < $9::timestamptz + AND client_id = $2 + AND connect_at >= $8::timestamptz + AND connect_at < $9::timestamptz );`; await this.db.query({ @@ -227,10 +225,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor { type: 1184, value: lt } ] }); - await this.db.query('COMMIT'); } catch (error) { console.log(error); - await this.db.query('ROLLBACK'); } } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { From 42ddacf28b9844de145245cff20ed27602207f46 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 16:46:58 +0200 Subject: [PATCH 100/169] pg-wire frustrations --- .../storage/PostgresReportStorageFactory.ts | 94 ++++++++++++------- 1 file changed, 59 insertions(+), 35 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 47ffd5978..c26849d5a 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -186,46 +186,70 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const connectIsoString = connect_at.toISOString(); const jwtExpIsoString = jwt_exp!.toISOString(); const { gte, lt } = this.updateTableFilter(); + const param = { + user_id: { type: 'varchar', value: user_id }, + client_id: { type: 'varchar', value: client_id }, + connect_at: { type: 1184, value: connectIsoString }, + sdk: { type: 'varchar', value: sdk }, + user_agent: { type: 'varchar', value: user_agent }, + jwt_exp: { type: 1184, value: jwtExpIsoString }, + id: { type: 'varchar', value: v4() }, + gte: { type: 1184, value: gte }, + lt: { type: 1184, value: lt } + }; + await this.db.query('BEGIN;'); try { - const query = ` - UPDATE sdk_report_events - SET connect_at = $3::timestamptz, - sdk = $4, - user_agent = $5, - jwt_exp = $6::timestamptz, - disconnect_at = NULL - WHERE user_id = $1 - AND client_id = $2 - AND connect_at >= $8::timestamptz - AND connect_at < $9::timestamptz; - - INSERT INTO sdk_report_events ( - user_id, client_id, connect_at, sdk, user_agent, jwt_exp, id - ) - SELECT $1, $2, $3::timestamptz, $4, $5, $6::timestamptz, $7 - WHERE NOT EXISTS ( - SELECT 1 FROM sdk_report_events - WHERE user_id = $1 - AND client_id = $2 - AND connect_at >= $8::timestamptz - AND connect_at < $9::timestamptz - );`; - - await this.db.query({ - statement: query, + const result = await this.db.query({ + statement: ` + UPDATE sdk_report_events + SET connect_at = $1::timestamptz, + sdk = $2, + user_agent = $3, + jwt_exp = $4::timestamptz, + disconnect_at = NULL + WHERE user_id = $5 + AND client_id = $6 + AND connect_at >= $7::timestamptz + AND connect_at < $8::timestamptz;`, params: [ - { type: 'varchar', value: user_id }, - { type: 'varchar', value: client_id }, - { type: 1184, value: connectIsoString }, - { type: 'varchar', value: sdk }, - { type: 'varchar', value: user_agent }, - { type: 1184, value: jwtExpIsoString }, - { type: 'varchar', value: v4() }, - { type: 1184, value: gte }, - { type: 1184, value: lt } + param.connect_at, + param.sdk, + param.user_agent, + param.jwt_exp, + param.user_id, + param.client_id, + param.gte, + param.lt ] }); + if (result.rowCount === 0) { + await this.db.query({ + statement: ` + INSERT INTO sdk_report_events ( + user_id, client_id, connect_at, sdk, user_agent, jwt_exp, id + ) + SELECT $1, $2, $3::timestamptz, $4, $5, $6::timestamptz, $7 + WHERE NOT EXISTS ( + SELECT 1 FROM sdk_report_events + WHERE user_id = $1 + AND client_id = $2 + AND connect_at >= $8::timestamptz + AND connect_at < $9::timestamptz + );`, + params: [ + param.user_id, + param.client_id, + param.connect_at, + param.sdk, + param.user_agent, + param.jwt_exp, + param.id + ] + }); + await this.db.query('COMMIT;'); + } } catch (error) { + await this.db.query('ROLLBACK;'); console.log(error); } } From 85d6e6b8ef83aab624c9084327f385b74f2d7cff Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 17:02:46 +0200 Subject: [PATCH 101/169] pg-wire frustrations --- .../storage/PostgresReportStorageFactory.ts | 53 ++++++++----------- 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index c26849d5a..d86591151 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -186,17 +186,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const connectIsoString = connect_at.toISOString(); const jwtExpIsoString = jwt_exp!.toISOString(); const { gte, lt } = this.updateTableFilter(); - const param = { - user_id: { type: 'varchar', value: user_id }, - client_id: { type: 'varchar', value: client_id }, - connect_at: { type: 1184, value: connectIsoString }, - sdk: { type: 'varchar', value: sdk }, - user_agent: { type: 'varchar', value: user_agent }, - jwt_exp: { type: 1184, value: jwtExpIsoString }, - id: { type: 'varchar', value: v4() }, - gte: { type: 1184, value: gte }, - lt: { type: 1184, value: lt } - }; + const uuid = v4(); await this.db.query('BEGIN;'); try { const result = await this.db.query({ @@ -210,20 +200,22 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor WHERE user_id = $5 AND client_id = $6 AND connect_at >= $7::timestamptz - AND connect_at < $8::timestamptz;`, + AND connect_at < $8::timestamptz; +`, params: [ - param.connect_at, - param.sdk, - param.user_agent, - param.jwt_exp, - param.user_id, - param.client_id, - param.gte, - param.lt + { type: 1184, value: connectIsoString }, + { type: 'varchar', value: sdk }, + { type: 'varchar', value: user_agent }, + { type: 1184, value: jwtExpIsoString }, + { type: 'varchar', value: user_id }, + { type: 'varchar', value: client_id }, + { type: 1184, value: gte }, + { type: 1184, value: lt } ] }); - if (result.rowCount === 0) { - await this.db.query({ + console.log(result.rows); + if (result.rows.length === 0) { + const result = await this.db.query({ statement: ` INSERT INTO sdk_report_events ( user_id, client_id, connect_at, sdk, user_agent, jwt_exp, id @@ -237,17 +229,18 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor AND connect_at < $9::timestamptz );`, params: [ - param.user_id, - param.client_id, - param.connect_at, - param.sdk, - param.user_agent, - param.jwt_exp, - param.id + { type: 'varchar', value: user_id }, + { type: 'varchar', value: client_id }, + { type: 1184, value: connectIsoString }, + { type: 'varchar', value: sdk }, + { type: 'varchar', value: user_agent }, + { type: 1184, value: jwtExpIsoString }, + { type: 'varchar', value: uuid } ] }); - await this.db.query('COMMIT;'); + console.log(result.rows); } + await this.db.query('COMMIT;'); } catch (error) { await this.db.query('ROLLBACK;'); console.log(error); From 052ca630e06eb1a8f03d45eebd5498239739a538 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 17:12:36 +0200 Subject: [PATCH 102/169] pg-wire frustrations --- .../src/storage/PostgresReportStorageFactory.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index d86591151..c11bf6094 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -235,7 +235,9 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor { type: 'varchar', value: sdk }, { type: 'varchar', value: user_agent }, { type: 1184, value: jwtExpIsoString }, - { type: 'varchar', value: uuid } + { type: 'varchar', value: uuid }, + { type: 1184, value: gte }, + { type: 1184, value: lt } ] }); console.log(result.rows); From d4118ac9ad7870591018df59f77ad6d36c29c1ec Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 17:13:20 +0200 Subject: [PATCH 103/169] pg-wire frustrations --- .../src/storage/PostgresReportStorageFactory.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index c11bf6094..be4f73a05 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -187,7 +187,6 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const jwtExpIsoString = jwt_exp!.toISOString(); const { gte, lt } = this.updateTableFilter(); const uuid = v4(); - await this.db.query('BEGIN;'); try { const result = await this.db.query({ statement: ` @@ -242,9 +241,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor }); console.log(result.rows); } - await this.db.query('COMMIT;'); } catch (error) { - await this.db.query('ROLLBACK;'); console.log(error); } } From 2c1eb1b85a4fd78ff8e170a7cc5aff01df98fc11 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 17:30:45 +0200 Subject: [PATCH 104/169] pg-wire frustrations --- .../storage/PostgresReportStorageFactory.ts | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index be4f73a05..cb896456e 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -130,11 +130,11 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor FROM filtered GROUP BY sdk ) - SELECT - COALESCE(u.count, 0) AS users, - JSON_AGG(ROW_TO_JSON(s)) AS sdks - FROM unique_users u - JOIN sdk_versions_array s ON TRUE; +-- SELECT +-- COALESCE(u.count, 0) AS users, +-- JSON_AGG(ROW_TO_JSON(s)) AS sdks +-- FROM unique_users u +-- JOIN sdk_versions_array s ON TRUE; `; return { statement: query @@ -163,13 +163,19 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor FROM filtered GROUP BY sdk ) - SELECT COALESCE(u.count, 0) AS users, JSON_AGG(ROW_TO_JSON(s)) AS sdks - FROM unique_users u - JOIN sdk_versions_array s ON TRUE; +-- SELECT +-- (SELECT COALESCE(count, 0) FROM unique_users) AS users, +-- (SELECT JSON_AGG(ROW_TO_JSON(s)) FROM sdk_versions_array s) AS sdks; `; const lt = endDate.toISOString(); const gt = startDate.toISOString(); - return { statement: query, params: [{ value: gt }, { value: lt }] }; + return { + statement: query, + params: [ + { type: 1184, value: gt }, + { type: 1184, value: lt } + ] + }; } private updateTableFilter() { @@ -278,7 +284,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { const statement = this.listConnectionsDateRangeQuery(data); const result = await this.db.query(statement); - console.log(result.rows); + console.log(result); return { users: 0, sdks: [] @@ -331,7 +337,6 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor `; const params = [{ value: lt }]; const result = await this.db.query({ statement: query, params }); - console.log(result.rows); } async [Symbol.asyncDispose]() { From 3f8cec5a2c772d43214776cb0ceeeb5b9630b56b Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 17:38:44 +0200 Subject: [PATCH 105/169] pg-wire frustrations --- .../src/storage/PostgresReportStorageFactory.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index cb896456e..2424c73e3 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -130,11 +130,9 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor FROM filtered GROUP BY sdk ) --- SELECT --- COALESCE(u.count, 0) AS users, --- JSON_AGG(ROW_TO_JSON(s)) AS sdks --- FROM unique_users u --- JOIN sdk_versions_array s ON TRUE; + SELECT + (SELECT COALESCE(count, 0) FROM unique_users) AS users, + (SELECT JSON_AGG(ROW_TO_JSON(s)) FROM sdk_versions_array s) AS sdks; `; return { statement: query @@ -163,9 +161,9 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor FROM filtered GROUP BY sdk ) --- SELECT --- (SELECT COALESCE(count, 0) FROM unique_users) AS users, --- (SELECT JSON_AGG(ROW_TO_JSON(s)) FROM sdk_versions_array s) AS sdks; + SELECT + (SELECT COALESCE(count, 0) FROM unique_users) AS users, + (SELECT JSON_AGG(ROW_TO_JSON(s)) FROM sdk_versions_array s) AS sdks; `; const lt = endDate.toISOString(); const gt = startDate.toISOString(); From f824035f8385470dfea65b1d5b7bf35a0f0313c9 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 17:44:54 +0200 Subject: [PATCH 106/169] pg-wire frustrations --- .../src/storage/PostgresReportStorageFactory.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 2424c73e3..211990409 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -282,7 +282,14 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { const statement = this.listConnectionsDateRangeQuery(data); const result = await this.db.query(statement); - console.log(result); + console.log( + result.results.map((result) => { + if (result.rows.length > 0) { + console.log(result.columns); + console.log(result.rows); + } + }) + ); return { users: 0, sdks: [] From 27eb8f4dabe68b02538b643d143e67c1827f718a Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 29 Jul 2025 17:52:50 +0200 Subject: [PATCH 107/169] pg-wire frustrations --- .../storage/PostgresReportStorageFactory.ts | 89 +++++++++---------- 1 file changed, 40 insertions(+), 49 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 211990409..28b0473b7 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -191,9 +191,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const jwtExpIsoString = jwt_exp!.toISOString(); const { gte, lt } = this.updateTableFilter(); const uuid = v4(); - try { - const result = await this.db.query({ - statement: ` + const result = await this.db.query({ + statement: ` UPDATE sdk_report_events SET connect_at = $1::timestamptz, sdk = $2, @@ -205,21 +204,20 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor AND connect_at >= $7::timestamptz AND connect_at < $8::timestamptz; `, - params: [ - { type: 1184, value: connectIsoString }, - { type: 'varchar', value: sdk }, - { type: 'varchar', value: user_agent }, - { type: 1184, value: jwtExpIsoString }, - { type: 'varchar', value: user_id }, - { type: 'varchar', value: client_id }, - { type: 1184, value: gte }, - { type: 1184, value: lt } - ] - }); - console.log(result.rows); - if (result.rows.length === 0) { - const result = await this.db.query({ - statement: ` + params: [ + { type: 1184, value: connectIsoString }, + { type: 'varchar', value: sdk }, + { type: 'varchar', value: user_agent }, + { type: 1184, value: jwtExpIsoString }, + { type: 'varchar', value: user_id }, + { type: 'varchar', value: client_id }, + { type: 1184, value: gte }, + { type: 1184, value: lt } + ] + }); + if (result.rows.length === 0) { + await this.db.query({ + statement: ` INSERT INTO sdk_report_events ( user_id, client_id, connect_at, sdk, user_agent, jwt_exp, id ) @@ -231,22 +229,18 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor AND connect_at >= $8::timestamptz AND connect_at < $9::timestamptz );`, - params: [ - { type: 'varchar', value: user_id }, - { type: 'varchar', value: client_id }, - { type: 1184, value: connectIsoString }, - { type: 'varchar', value: sdk }, - { type: 'varchar', value: user_agent }, - { type: 1184, value: jwtExpIsoString }, - { type: 'varchar', value: uuid }, - { type: 1184, value: gte }, - { type: 1184, value: lt } - ] - }); - console.log(result.rows); - } - } catch (error) { - console.log(error); + params: [ + { type: 'varchar', value: user_id }, + { type: 'varchar', value: client_id }, + { type: 1184, value: connectIsoString }, + { type: 'varchar', value: sdk }, + { type: 'varchar', value: user_agent }, + { type: 1184, value: jwtExpIsoString }, + { type: 'varchar', value: uuid }, + { type: 1184, value: gte }, + { type: 1184, value: lt } + ] + }); } } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { @@ -263,21 +257,16 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor AND connect_at >= CAST($4 AS TIMESTAMP WITH TIME ZONE) AND connect_at < CAST($5 AS TIMESTAMP WITH TIME ZONE);`; - try { - const result = await this.db.query({ - statement: query, - params: [ - { type: 1184, value: disconnectIsoString }, - { type: 'varchar', value: user_id }, - { type: 'varchar', value: client_id }, - { type: 1184, value: gte }, - { type: 1184, value: lt } - ] - }); - console.log(result.rows); - } catch (error) { - console.log(error); - } + const result = await this.db.query({ + statement: query, + params: [ + { type: 1184, value: disconnectIsoString }, + { type: 'varchar', value: user_id }, + { type: 'varchar', value: client_id }, + { type: 1184, value: gte }, + { type: 1184, value: lt } + ] + }); } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { const statement = this.listConnectionsDateRangeQuery(data); @@ -290,6 +279,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } }) ); + const parsedResult = result.results[1].rows[1]; + console.log(parsedResult); return { users: 0, sdks: [] From 02a50368ed9b270095c5191f0374af967868e318 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 08:41:30 +0200 Subject: [PATCH 108/169] change the sql query to test --- .../storage/PostgresReportStorageFactory.ts | 150 ++++++++++-------- 1 file changed, 84 insertions(+), 66 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 28b0473b7..2b63929b3 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -108,72 +108,100 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } } - private listConnectionsDateRangeQuery(data: event_types.ListCurrentConnectionsRequest) { + private async listConnectionsDateRangeQuery(data: event_types.ListCurrentConnectionsRequest) { const { range } = data; if (!range) { - const query = ` - WITH filtered AS ( - SELECT * - FROM sdk_report_events - WHERE disconnect_at IS NULL + return this.db.sql` + WITH + filtered AS ( + SELECT + * + FROM + sdk_report_events + WHERE + disconnect_at IS NULL AND jwt_exp > NOW() - ), - unique_users AS ( - SELECT COUNT(DISTINCT user_id) AS count - FROM filtered - ), - sdk_versions_array AS ( - SELECT sdk, + ), + unique_users AS ( + SELECT + COUNT(DISTINCT user_id) AS count + FROM + filtered + ), + sdk_versions_array AS ( + SELECT + sdk, COUNT(*) AS total, COUNT(DISTINCT client_id) AS clients, COUNT(DISTINCT user_id) AS users - FROM filtered - GROUP BY sdk - ) + FROM + filtered + GROUP BY + sdk + ) SELECT - (SELECT COALESCE(count, 0) FROM unique_users) AS users, - (SELECT JSON_AGG(ROW_TO_JSON(s)) FROM sdk_versions_array s) AS sdks; - `; - return { - statement: query - }; + ( + SELECT + COALESCE(count, 0) + FROM + unique_users + ) AS users, + ( + SELECT + JSON_AGG(ROW_TO_JSON(s)) + FROM + sdk_versions_array s + ) AS sdks; + `.first(); } const endDate = data.range?.end_date ? new Date(data.range.end_date) : new Date(); const startDate = new Date(range.start_date); - const query = ` - WITH filtered AS ( - SELECT * - FROM sdk_report_events - WHERE disconnect_at IS NULL - AND jwt_exp > NOW() - AND connect_at > CAST($1 AS TIMESTAMP WITH TIME ZONE) - AND connect_at <= CAST($2 AS TIMESTAMP WITH TIME ZONE) - ), - unique_users AS ( - SELECT COUNT(DISTINCT user_id) AS count - FROM filtered - ), - sdk_versions_array AS ( - SELECT sdk, - COUNT(*) AS total, - COUNT(DISTINCT client_id) AS clients, - COUNT(DISTINCT user_id) AS users - FROM filtered - GROUP BY sdk - ) - SELECT - (SELECT COALESCE(count, 0) FROM unique_users) AS users, - (SELECT JSON_AGG(ROW_TO_JSON(s)) FROM sdk_versions_array s) AS sdks; - `; const lt = endDate.toISOString(); const gt = startDate.toISOString(); - return { - statement: query, - params: [ - { type: 1184, value: gt }, - { type: 1184, value: lt } - ] - }; + return await this.db.sql` + WITH + filtered AS ( + SELECT + * + FROM + sdk_report_events + WHERE + disconnect_at IS NULL + AND jwt_exp > NOW() + AND connect_at > ${{ type: 1184, value: gt }} + AND connect_at <= ${{ type: 1184, value: lt }} + ), + unique_users AS ( + SELECT + COUNT(DISTINCT user_id) AS count + FROM + filtered + ), + sdk_versions_array AS ( + SELECT + sdk, + COUNT(*) AS total, + COUNT(DISTINCT client_id) AS clients, + COUNT(DISTINCT user_id) AS users + FROM + filtered + GROUP BY + sdk + ) + SELECT + ( + SELECT + COALESCE(count, 0) + FROM + unique_users + ) AS users, + ( + SELECT + JSON_AGG(ROW_TO_JSON(s)) + FROM + sdk_versions_array s + ) AS sdks; + `.first(); } private updateTableFilter() { @@ -269,18 +297,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor }); } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { - const statement = this.listConnectionsDateRangeQuery(data); - const result = await this.db.query(statement); - console.log( - result.results.map((result) => { - if (result.rows.length > 0) { - console.log(result.columns); - console.log(result.rows); - } - }) - ); - const parsedResult = result.results[1].rows[1]; - console.log(parsedResult); + const rows = await this.listConnectionsDateRangeQuery(data); + console.log(rows); return { users: 0, sdks: [] From c8c38dfd9f0fbd85039fb4e4c90f80ef43143783 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 09:14:36 +0200 Subject: [PATCH 109/169] testing --- .../src/storage/PostgresReportStorageFactory.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 2b63929b3..b0a5f4440 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -297,12 +297,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor }); } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { - const rows = await this.listConnectionsDateRangeQuery(data); - console.log(rows); - return { - users: 0, - sdks: [] - }; + // @ts-ignore + return await this.listConnectionsDateRangeQuery(data); } async scrapeSdkData(data: ScrapeSdkDataRequest): Promise { From d039559890bb98875f388430574431cfde94e12d Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 09:21:10 +0200 Subject: [PATCH 110/169] testing --- .../src/storage/PostgresReportStorageFactory.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index b0a5f4440..5d8ce0be6 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -297,8 +297,14 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor }); } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { + const rows = await this.listConnectionsDateRangeQuery(data); // @ts-ignore - return await this.listConnectionsDateRangeQuery(data); + console.log(rows.sdks.data); + return { + // @ts-ignore + users: Number(rows.users), + sdks: [] + }; } async scrapeSdkData(data: ScrapeSdkDataRequest): Promise { From aefa0f9d58f59fb5b56048d879379b211160f46b Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 09:29:49 +0200 Subject: [PATCH 111/169] testing --- .../src/storage/PostgresReportStorageFactory.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 5d8ce0be6..1e8bf9f7b 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -298,8 +298,9 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { const rows = await this.listConnectionsDateRangeQuery(data); + console.log({ rows }); // @ts-ignore - console.log(rows.sdks.data); + console.log(JSON.stringify(rows.sdks, null, 2)); return { // @ts-ignore users: Number(rows.users), From e096409ca43868c912f1ab314b6cf28eb8443d91 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 09:48:53 +0200 Subject: [PATCH 112/169] testing --- .../src/storage/PostgresReportStorageFactory.ts | 11 ++++++++--- .../src/types/models/SdkReporting.ts | 11 +++++++++++ .../src/types/models/models-index.ts | 1 + 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 modules/module-postgres-storage/src/types/models/SdkReporting.ts diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 1e8bf9f7b..d01bb2d95 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -15,6 +15,7 @@ import { SdkConnectBucketData, SdkDisconnectEventData } from '@powersync/service-types/src/events.js'; +import { SdkReporting } from '../types/models/SdkReporting.js'; export type PostgresReportStorageOptions = { config: NormalizedPostgresStorageConfig; @@ -152,7 +153,9 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor FROM sdk_versions_array s ) AS sdks; - `.first(); + ` + .decoded(SdkReporting) + .first(); } const endDate = data.range?.end_date ? new Date(data.range.end_date) : new Date(); const startDate = new Date(range.start_date); @@ -201,7 +204,9 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor FROM sdk_versions_array s ) AS sdks; - `.first(); + ` + .decoded(SdkReporting) + .first(); } private updateTableFilter() { @@ -300,7 +305,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const rows = await this.listConnectionsDateRangeQuery(data); console.log({ rows }); // @ts-ignore - console.log(JSON.stringify(rows.sdks, null, 2)); + console.log(rows.sdks); return { // @ts-ignore users: Number(rows.users), diff --git a/modules/module-postgres-storage/src/types/models/SdkReporting.ts b/modules/module-postgres-storage/src/types/models/SdkReporting.ts new file mode 100644 index 000000000..3bd602373 --- /dev/null +++ b/modules/module-postgres-storage/src/types/models/SdkReporting.ts @@ -0,0 +1,11 @@ +import * as t from 'ts-codec'; +import { bigint, jsonb } from '../codecs.js'; + +export const SdkReporting = t.object({ + users: bigint, + sdks: t.object({ + data: jsonb(t.record(t.string)) + }) +}); + +export type SdkReporting = t.Encoded; diff --git a/modules/module-postgres-storage/src/types/models/models-index.ts b/modules/module-postgres-storage/src/types/models/models-index.ts index fb5574608..81a528d33 100644 --- a/modules/module-postgres-storage/src/types/models/models-index.ts +++ b/modules/module-postgres-storage/src/types/models/models-index.ts @@ -8,3 +8,4 @@ export * from './Migration.js'; export * from './SourceTable.js'; export * from './SyncRules.js'; export * from './WriteCheckpoint.js'; +export * from './SdkReporting.js'; From 7526ab2132e2c3fe203bfcbf9ad286af8a7086af Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 10:02:26 +0200 Subject: [PATCH 113/169] ts-codecs --- .../src/storage/PostgresReportStorageFactory.ts | 2 -- .../src/types/models/SdkReporting.ts | 8 +++++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index d01bb2d95..7c7febc30 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -132,7 +132,6 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor sdk_versions_array AS ( SELECT sdk, - COUNT(*) AS total, COUNT(DISTINCT client_id) AS clients, COUNT(DISTINCT user_id) AS users FROM @@ -183,7 +182,6 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor sdk_versions_array AS ( SELECT sdk, - COUNT(*) AS total, COUNT(DISTINCT client_id) AS clients, COUNT(DISTINCT user_id) AS users FROM diff --git a/modules/module-postgres-storage/src/types/models/SdkReporting.ts b/modules/module-postgres-storage/src/types/models/SdkReporting.ts index 3bd602373..b96a8841b 100644 --- a/modules/module-postgres-storage/src/types/models/SdkReporting.ts +++ b/modules/module-postgres-storage/src/types/models/SdkReporting.ts @@ -1,10 +1,16 @@ import * as t from 'ts-codec'; import { bigint, jsonb } from '../codecs.js'; +export const Sdks = t.object({ + sdk: t.string, + clients: t.number, + users: t.number +}); + export const SdkReporting = t.object({ users: bigint, sdks: t.object({ - data: jsonb(t.record(t.string)) + data: jsonb(t.array(Sdks)) }) }); From f1244fdf72fdef07b2fad6017eed45efd5961af4 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 10:25:52 +0200 Subject: [PATCH 114/169] ts-codecs --- .../storage/PostgresReportStorageFactory.ts | 27 +++++++++++-------- .../src/types/models/SdkReporting.ts | 5 +++- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 7c7febc30..dbd8c34b1 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -15,7 +15,7 @@ import { SdkConnectBucketData, SdkDisconnectEventData } from '@powersync/service-types/src/events.js'; -import { SdkReporting } from '../types/models/SdkReporting.js'; +import { SdkReporting, SdkReportingDecoded } from '../types/models/SdkReporting.js'; export type PostgresReportStorageOptions = { config: NormalizedPostgresStorageConfig; @@ -109,7 +109,13 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } } - private async listConnectionsDateRangeQuery(data: event_types.ListCurrentConnectionsRequest) { + private mapListCurrentConnectionsResponse(result: SdkReportingDecoded): ListCurrentConnections { + return { + users: Number(result.users), + sdks: result.sdks.data + }; + } + private async listConnectionsQuery(data: event_types.ListCurrentConnectionsRequest) { const { range } = data; if (!range) { return this.db.sql` @@ -300,15 +306,14 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor }); } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { - const rows = await this.listConnectionsDateRangeQuery(data); - console.log({ rows }); - // @ts-ignore - console.log(rows.sdks); - return { - // @ts-ignore - users: Number(rows.users), - sdks: [] - }; + const result = await this.listConnectionsQuery(data); + if (!result) { + return { + users: 0, + sdks: [] + }; + } + return this.mapListCurrentConnectionsResponse(result); } async scrapeSdkData(data: ScrapeSdkDataRequest): Promise { diff --git a/modules/module-postgres-storage/src/types/models/SdkReporting.ts b/modules/module-postgres-storage/src/types/models/SdkReporting.ts index b96a8841b..ee54bd0c8 100644 --- a/modules/module-postgres-storage/src/types/models/SdkReporting.ts +++ b/modules/module-postgres-storage/src/types/models/SdkReporting.ts @@ -7,11 +7,14 @@ export const Sdks = t.object({ users: t.number }); +export type Sdks = t.Encoded; + export const SdkReporting = t.object({ users: bigint, sdks: t.object({ - data: jsonb(t.array(Sdks)) + data: jsonb(t.array(Sdks)) }) }); export type SdkReporting = t.Encoded; +export type SdkReportingDecoded = t.Decoded; From f88544f3785638606c434bf4d5d7dae0eb6aae35 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 10:33:30 +0200 Subject: [PATCH 115/169] ts-codecs --- .../src/storage/PostgresReportStorageFactory.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index dbd8c34b1..4731f8248 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -112,7 +112,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor private mapListCurrentConnectionsResponse(result: SdkReportingDecoded): ListCurrentConnections { return { users: Number(result.users), - sdks: result.sdks.data + sdks: [] }; } private async listConnectionsQuery(data: event_types.ListCurrentConnectionsRequest) { @@ -313,6 +313,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor sdks: [] }; } + console.log(result); return this.mapListCurrentConnectionsResponse(result); } From 6b5b58b917f450436f30d24d0411e7d64f468262 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 10:39:20 +0200 Subject: [PATCH 116/169] ts-codecs --- .../src/storage/PostgresReportStorageFactory.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 4731f8248..ce52f5274 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -110,6 +110,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } private mapListCurrentConnectionsResponse(result: SdkReportingDecoded): ListCurrentConnections { + console.log(result.sdks); return { users: Number(result.users), sdks: [] @@ -313,7 +314,6 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor sdks: [] }; } - console.log(result); return this.mapListCurrentConnectionsResponse(result); } @@ -362,7 +362,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor ); `; const params = [{ value: lt }]; - const result = await this.db.query({ statement: query, params }); + await this.db.query({ statement: query, params }); } async [Symbol.asyncDispose]() { From 792b2995cab830d34fd0ecdc549f8afd4b68f22f Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 10:45:38 +0200 Subject: [PATCH 117/169] ts-codecs --- .../src/storage/PostgresReportStorageFactory.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index ce52f5274..4f07c27b2 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -110,10 +110,9 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } private mapListCurrentConnectionsResponse(result: SdkReportingDecoded): ListCurrentConnections { - console.log(result.sdks); return { users: Number(result.users), - sdks: [] + sdks: result.sdks?.data || [] }; } private async listConnectionsQuery(data: event_types.ListCurrentConnectionsRequest) { From caf0c2633c5df328976d4ca0534e35af25c552d2 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 11:13:08 +0200 Subject: [PATCH 118/169] ts-codecs --- .../storage/PostgresReportStorageFactory.ts | 220 +++++++++--------- 1 file changed, 109 insertions(+), 111 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 4f07c27b2..48437d623 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -109,7 +109,13 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } } - private mapListCurrentConnectionsResponse(result: SdkReportingDecoded): ListCurrentConnections { + private mapListCurrentConnectionsResponse(result: SdkReportingDecoded | null): ListCurrentConnections { + if (!result) { + return { + users: 0, + sdks: [] + }; + } return { users: Number(result.users), sdks: result.sdks?.data || [] @@ -228,140 +234,132 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const jwtExpIsoString = jwt_exp!.toISOString(); const { gte, lt } = this.updateTableFilter(); const uuid = v4(); - const result = await this.db.query({ - statement: ` - UPDATE sdk_report_events - SET connect_at = $1::timestamptz, - sdk = $2, - user_agent = $3, - jwt_exp = $4::timestamptz, - disconnect_at = NULL - WHERE user_id = $5 - AND client_id = $6 - AND connect_at >= $7::timestamptz - AND connect_at < $8::timestamptz; -`, - params: [ - { type: 1184, value: connectIsoString }, - { type: 'varchar', value: sdk }, - { type: 'varchar', value: user_agent }, - { type: 1184, value: jwtExpIsoString }, - { type: 'varchar', value: user_id }, - { type: 'varchar', value: client_id }, - { type: 1184, value: gte }, - { type: 1184, value: lt } - ] - }); - if (result.rows.length === 0) { - await this.db.query({ - statement: ` - INSERT INTO sdk_report_events ( - user_id, client_id, connect_at, sdk, user_agent, jwt_exp, id - ) - SELECT $1, $2, $3::timestamptz, $4, $5, $6::timestamptz, $7 - WHERE NOT EXISTS ( - SELECT 1 FROM sdk_report_events - WHERE user_id = $1 - AND client_id = $2 - AND connect_at >= $8::timestamptz - AND connect_at < $9::timestamptz - );`, - params: [ - { type: 'varchar', value: user_id }, - { type: 'varchar', value: client_id }, - { type: 1184, value: connectIsoString }, - { type: 'varchar', value: sdk }, - { type: 'varchar', value: user_agent }, - { type: 1184, value: jwtExpIsoString }, - { type: 'varchar', value: uuid }, - { type: 1184, value: gte }, - { type: 1184, value: lt } - ] - }); + const result = await this.db.sql` + UPDATE sdk_report_events + SET + connect_at = ${{ type: 1184, value: connectIsoString }}, + sdk = ${{ type: 'varchar', value: sdk }}, + user_agent = ${{ type: 'varchar', value: user_agent }}, + jwt_exp = ${{ type: 1184, value: jwtExpIsoString }}, + disconnect_at = NULL + WHERE + user_id = ${{ type: 'varchar', value: user_id }} + AND client_id = ${{ type: 'varchar', value: client_id }} + AND connect_at >= ${{ type: 1184, value: gte }} + AND connect_at < ${{ type: 1184, value: lt }}; + `.rows(); + if (result.length === 0) { + await this.db.sql` + INSERT INTO + sdk_report_events ( + user_id, + client_id, + connect_at, + sdk, + user_agent, + jwt_exp, + id + ) + SELECT + ${{ type: 'varchar', value: user_id }}, + ${{ type: 'varchar', value: client_id }}, + ${{ type: 1184, value: connectIsoString }}, + ${{ type: 'varchar', value: sdk }}, + ${{ type: 'varchar', value: user_agent }}, + ${{ type: 1184, value: jwtExpIsoString }}, + ${{ type: 'varchar', value: uuid }} + WHERE + NOT EXISTS ( + SELECT + 1 + FROM + sdk_report_events + WHERE + user_id = ${{ type: 'varchar', value: user_id }} + AND client_id = ${{ type: 'varchar', value: client_id }} + AND connect_at >= ${{ type: 1184, value: gte }} + AND connect_at < ${{ type: 1184, value: lt }} + ); + `.execute(); } } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { const { user_id, client_id, disconnect_at } = data; const disconnectIsoString = disconnect_at.toISOString(); const { gte, lt } = this.updateTableFilter(); - const query = ` + await this.db.sql` UPDATE sdk_report_events SET - disconnect_at = CAST($1 AS TIMESTAMP WITH TIME ZONE), + disconnect_at = ${{ type: 1184, value: disconnectIsoString }}, jwt_exp = NULL - WHERE user_id = $2 - AND client_id = $3 - AND connect_at >= CAST($4 AS TIMESTAMP WITH TIME ZONE) - AND connect_at < CAST($5 AS TIMESTAMP WITH TIME ZONE);`; - - const result = await this.db.query({ - statement: query, - params: [ - { type: 1184, value: disconnectIsoString }, - { type: 'varchar', value: user_id }, - { type: 'varchar', value: client_id }, - { type: 1184, value: gte }, - { type: 1184, value: lt } - ] - }); + WHERE + user_id = ${{ type: 'varchar', value: user_id }} + AND client_id = ${{ type: 'varchar', value: client_id }} + AND connect_at >= ${{ type: 1184, value: gte }} + AND connect_at < ${{ type: 1184, value: lt }}; + `.execute(); } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { const result = await this.listConnectionsQuery(data); - if (!result) { - return { - users: 0, - sdks: [] - }; - } return this.mapListCurrentConnectionsResponse(result); } async scrapeSdkData(data: ScrapeSdkDataRequest): Promise { const { timeframe, interval } = data; const { lt, gt } = this.timeFrameQuery(timeframe, interval); - const query = ` - WITH filtered AS ( - SELECT * - FROM sdk_report_events - WHERE connect_at > CAST($1 AS TIMESTAMP WITH TIME ZONE) - AND connect_at <= CAST($2 AS TIMESTAMP WITH TIME ZONE) - ), - unique_users AS ( - SELECT COUNT(DISTINCT user_id) AS count - FROM filtered - ), - sdk_versions_array AS ( - SELECT sdk, - COUNT(*) AS total, + const result = await this.db.sql` + WITH + filtered AS ( + SELECT + * + FROM + sdk_report_events + WHERE + connect_at > ${{ type: 1184, value: gt }} + AND connect_at <= ${{ type: 1184, value: lt }} + ), + unique_users AS ( + SELECT + COUNT(DISTINCT user_id) AS count + FROM + filtered + ), + sdk_versions_array AS ( + SELECT + sdk, COUNT(DISTINCT client_id) AS clients, COUNT(DISTINCT user_id) AS users - FROM filtered - GROUP BY sdk - ) - SELECT COALESCE(u.count, 0) AS users, JSON_AGG(ROW_TO_JSON(s)) AS sdks - FROM unique_users u - JOIN sdk_versions_array s ON TRUE; - `; - const result = await this.db.query({ statement: query, params: [{ value: gt }, { value: lt }] }); - console.log(result.rows); - return { - users: 0, - sdks: [] - }; + FROM + filtered + GROUP BY + sdk + ) + SELECT + COALESCE(u.count, 0) AS users, + JSON_AGG(ROW_TO_JSON(s)) AS sdks + FROM + unique_users u + JOIN sdk_versions_array s ON TRUE; + ` + .decoded(SdkReporting) + .first(); + return this.mapListCurrentConnectionsResponse(result); } async deleteOldSdkData(data: DeleteOldSdkData): Promise { const { timeframe, interval } = data; const { lt } = this.timeFrameDeleteQuery(timeframe, interval); - const query = ` - DELETE FROM sdk_report_events - WHERE connect_at < CAST($1 AS TIMESTAMP WITH TIME ZONE) - AND ( - disconnect_at IS NOT NULL - OR (jwt_exp < NOW() AND disconnect_at IS NULL) - ); -`; - const params = [{ value: lt }]; - await this.db.query({ statement: query, params }); + await this.db.sql` + DELETE FROM sdk_report_events + WHERE + connect_at < ${{ type: 1184, value: lt }} + AND ( + disconnect_at IS NOT NULL + OR ( + jwt_exp < NOW() + AND disconnect_at IS NULL + ) + ); + `.execute(); } async [Symbol.asyncDispose]() { From b2d5e9e955e99e66e004f5e0f12ed726b31ec7d9 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 11:18:46 +0200 Subject: [PATCH 119/169] fixed sdk scrape query --- .../src/storage/PostgresReportStorageFactory.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 48437d623..47701e1a1 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -335,11 +335,18 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor sdk ) SELECT - COALESCE(u.count, 0) AS users, - JSON_AGG(ROW_TO_JSON(s)) AS sdks - FROM - unique_users u - JOIN sdk_versions_array s ON TRUE; + ( + SELECT + COALESCE(count, 0) + FROM + unique_users + ) AS users, + ( + SELECT + JSON_AGG(ROW_TO_JSON(s)) + FROM + sdk_versions_array s + ) AS sdks; ` .decoded(SdkReporting) .first(); From e538cdff055c576641e42e2de5a60907874141d6 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 11:31:50 +0200 Subject: [PATCH 120/169] checking delete old data --- .../src/storage/PostgresReportStorageFactory.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 47701e1a1..a5046687a 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -355,7 +355,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor async deleteOldSdkData(data: DeleteOldSdkData): Promise { const { timeframe, interval } = data; const { lt } = this.timeFrameDeleteQuery(timeframe, interval); - await this.db.sql` + const result = await this.db.sql` DELETE FROM sdk_report_events WHERE connect_at < ${{ type: 1184, value: lt }} @@ -367,6 +367,11 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor ) ); `.execute(); + if (result.rows.length > 0) { + result.rows.forEach((row) => { + console.log(row); + }); + } } async [Symbol.asyncDispose]() { From f385a50c2c430f245310809975c036562b856d79 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 11:39:43 +0200 Subject: [PATCH 121/169] checking delete old data --- .../src/storage/PostgresReportStorageFactory.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index a5046687a..77e0abc7b 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -367,11 +367,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor ) ); `.execute(); - if (result.rows.length > 0) { - result.rows.forEach((row) => { - console.log(row); - }); - } + console.log(result); } async [Symbol.asyncDispose]() { From 8bed767ab081b1bdc8ab527ccb8824cc04fa756e Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 11:54:37 +0200 Subject: [PATCH 122/169] deleted row logging --- .../src/storage/MongoReportStorage.ts | 4 +++- .../src/storage/PostgresReportStorageFactory.ts | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 3d75db0dc..3cc038eae 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -166,7 +166,9 @@ export class MongoReportStorage implements storage.ReportStorageFactory { $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }] }); if (result.deletedCount > 0) { - logger.info(`TTL: ${result.deletedCount} documents have been removed from sdk_report_events collection`); + logger.info( + `TTL ${interval}/${timeframe}: ${result.deletedCount} MongoDB documents have been removed from sdk_report_events.` + ); } } diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 77e0abc7b..9fa8ba49f 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -16,6 +16,8 @@ import { SdkDisconnectEventData } from '@powersync/service-types/src/events.js'; import { SdkReporting, SdkReportingDecoded } from '../types/models/SdkReporting.js'; +import { toInteger } from 'ix/util/tointeger.js'; +import { logger } from '@powersync/lib-services-framework'; export type PostgresReportStorageOptions = { config: NormalizedPostgresStorageConfig; @@ -367,7 +369,12 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor ) ); `.execute(); - console.log(result); + const deletedRows = toInteger(result.results[1].status.split(' ')[1] || '0'); + if (deletedRows > 0) { + logger.info( + `TTL ${interval}/${timeframe}: ${deletedRows} PostgresSQL rows have been removed from sdk_report_events.` + ); + } } async [Symbol.asyncDispose]() { From 551edf22c6e44b61bddfe0788c8ecc28416c0117 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 12:33:37 +0200 Subject: [PATCH 123/169] seperated migration for sdk --- .../migrations/scripts/1684951997326-init.ts | 22 --------- .../scripts/1753871047719-sdk_report.ts | 45 +++++++++++++++++++ 2 files changed, 45 insertions(+), 22 deletions(-) create mode 100644 modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index dc7f6d2ae..61e8699ea 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -128,28 +128,6 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { CONSTRAINT unique_user_sync PRIMARY KEY (user_id, sync_rules_id) ); `.execute(); - - await db.sql` - CREATE TABLE sdk_report_events ( - id TEXT PRIMARY KEY, - user_agent TEXT NOT NULL, - client_id TEXT NOT NULL, - user_id TEXT NOT NULL, - sdk TEXT NOT NULL, - jwt_exp TIMESTAMP WITH TIME ZONE, - connect_at TIMESTAMP WITH TIME ZONE NOT NULL, - disconnect_at TIMESTAMP WITH TIME ZONE, - CONSTRAINT unique_user_client_connect UNIQUE (user_id, client_id, connect_at) - ) - `.execute(); - - await db.sql` CREATE INDEX sdk_list_index ON sdk_report_events (connect_at, jwt_exp, disconnect_at) `.execute(); - - await db.sql` CREATE INDEX sdk_user_id_index ON sdk_report_events (user_id)`.execute(); - - await db.sql` CREATE INDEX sdk_client_id_index ON sdk_report_events (client_id)`.execute(); - - await db.sql` CREATE INDEX sdk_index ON sdk_report_events (sdk)`.execute(); }); }; diff --git a/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts b/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts new file mode 100644 index 000000000..0f155126e --- /dev/null +++ b/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts @@ -0,0 +1,45 @@ +import { migrations } from '@powersync/service-core'; + +import { openMigrationDB } from '../migration-utils.js'; + +export const up: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + await using client = openMigrationDB(configuration.storage); + + await client.transaction(async (db) => { + await db.sql` + CREATE TABLE IF NOT EXISTS sdk_report_events ( + id TEXT PRIMARY KEY, + user_agent TEXT NOT NULL, + client_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sdk TEXT NOT NULL, + jwt_exp TIMESTAMP WITH TIME ZONE, + connect_at TIMESTAMP WITH TIME ZONE NOT NULL, + disconnect_at TIMESTAMP WITH TIME ZONE, + ) + `.execute(); + + await db.sql` + CREATE INDEX IF NOT EXISTS sdk_list_index ON sdk_report_events (connect_at, jwt_exp, disconnect_at) + `.execute(); + + await db.sql`CREATE INDEX IF NOT EXISTS sdk_user_id_index ON sdk_report_events (user_id)`.execute(); + + await db.sql`CREATE INDEX IF NOT EXISTS sdk_client_id_index ON sdk_report_events (client_id)`.execute(); + + await db.sql`CREATE INDEX IF NOT EXISTS sdk_index ON sdk_report_events (sdk)`.execute(); + }); +}; + +export const down: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + await using client = openMigrationDB(configuration.storage); + client.lockConnection(async (db) => { + await db.sql`DROP TABLE IF EXISTS sdk_report_events`.execute(); + }); +}; From cabd182800d0acd95aae9bc13e476e5f93055b42 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 12:38:25 +0200 Subject: [PATCH 124/169] seperated migration for sdk --- .../src/migrations/scripts/1753871047719-sdk_report.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts b/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts index 0f155126e..c19f2f921 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts @@ -18,7 +18,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { sdk TEXT NOT NULL, jwt_exp TIMESTAMP WITH TIME ZONE, connect_at TIMESTAMP WITH TIME ZONE NOT NULL, - disconnect_at TIMESTAMP WITH TIME ZONE, + disconnect_at TIMESTAMP WITH TIME ZONE ) `.execute(); From 79043fa6999cc7551ed4fc7c7ca4a4ab7010719d Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 12:56:36 +0200 Subject: [PATCH 125/169] added redundancy for events --- packages/service-core/src/emitters/EmitterEngine.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index ff1932452..880c27ca8 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -29,6 +29,9 @@ export class EmitterEngine implements BaseEmitterEngine { } emit(event: K, data: event_types.SubscribeEvents[K]): void { + if (!this.events.has(event as event_types.EmitterEngineEvents)) { + return; + } this.emitter.emit(event, data); } From 45d17dfdcda2007c0cc48c8649901a919ee697e3 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 13:08:08 +0200 Subject: [PATCH 126/169] better log --- packages/service-core/src/emitters/EmitterEngine.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 880c27ca8..4ecee0fca 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -30,9 +30,10 @@ export class EmitterEngine implements BaseEmitterEngine { emit(event: K, data: event_types.SubscribeEvents[K]): void { if (!this.events.has(event as event_types.EmitterEngineEvents)) { - return; + this.emitter.emit('error', new Error(`There are no subscribed events for "${event}".`)); + } else { + this.emitter.emit(event, data); } - this.emitter.emit(event, data); } shutDown(): void { From c7cd9b1804d061730b9a3c59c9d651f7f333373f Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 13:15:41 +0200 Subject: [PATCH 127/169] oops --- packages/service-core/src/emitters/EmitterEngine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index 4ecee0fca..a2570050a 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -9,7 +9,7 @@ export class EmitterEngine implements BaseEmitterEngine { constructor() { this.emitter = new EventEmitter({ captureRejections: true }); this.emitter.on('error', (error: Error) => { - logger.error(error); + logger.error(error.message, { stack: error.stack }); }); } From 2b9695c1e32cdba677e64c5157e93d77042362c4 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 13:20:56 +0200 Subject: [PATCH 128/169] clean up --- packages/service-core/src/emitters/EmitterEngine.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EmitterEngine.ts index a2570050a..c5e7b6009 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EmitterEngine.ts @@ -9,7 +9,7 @@ export class EmitterEngine implements BaseEmitterEngine { constructor() { this.emitter = new EventEmitter({ captureRejections: true }); this.emitter.on('error', (error: Error) => { - logger.error(error.message, { stack: error.stack }); + logger.error(error.message); }); } @@ -29,11 +29,7 @@ export class EmitterEngine implements BaseEmitterEngine { } emit(event: K, data: event_types.SubscribeEvents[K]): void { - if (!this.events.has(event as event_types.EmitterEngineEvents)) { - this.emitter.emit('error', new Error(`There are no subscribed events for "${event}".`)); - } else { - this.emitter.emit(event, data); - } + this.emitter.emit(event, data); } shutDown(): void { From 2ff73da02868e882a9cad7c5cd2055375770fe6c Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 14:06:00 +0200 Subject: [PATCH 129/169] fixing a bug --- .../src/storage/PostgresReportStorageFactory.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 9fa8ba49f..3d50025f2 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -303,6 +303,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { const result = await this.listConnectionsQuery(data); + console.log(result); return this.mapListCurrentConnectionsResponse(result); } From 0c3a8531b9fb20b2fa967c8b18e6d844e1b820a7 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 14:16:07 +0200 Subject: [PATCH 130/169] fixing a bug --- .../src/storage/PostgresReportStorageFactory.ts | 1 - .../src/types/models/SdkReporting.ts | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 3d50025f2..9fa8ba49f 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -303,7 +303,6 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { const result = await this.listConnectionsQuery(data); - console.log(result); return this.mapListCurrentConnectionsResponse(result); } diff --git a/modules/module-postgres-storage/src/types/models/SdkReporting.ts b/modules/module-postgres-storage/src/types/models/SdkReporting.ts index ee54bd0c8..f44248bd3 100644 --- a/modules/module-postgres-storage/src/types/models/SdkReporting.ts +++ b/modules/module-postgres-storage/src/types/models/SdkReporting.ts @@ -11,9 +11,12 @@ export type Sdks = t.Encoded; export const SdkReporting = t.object({ users: bigint, - sdks: t.object({ - data: jsonb(t.array(Sdks)) - }) + sdks: t + .object({ + data: jsonb(t.array(Sdks)) + }) + .optional() + .or(t.Null) }); export type SdkReporting = t.Encoded; From 8277c8e8fbff2f4116df84234b7c17a4622b9483 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 17:00:45 +0200 Subject: [PATCH 131/169] removed query --- .../storage/PostgresReportStorageFactory.ts | 19 ++++++++++--------- .../PostgresTestStorageFactoryGenerator.ts | 16 ++++++++++++++++ .../test/src/sdk-report-storage.test.ts | 11 +++++++++++ .../module-postgres-storage/test/src/util.ts | 6 ++---- 4 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 modules/module-postgres-storage/test/src/sdk-report-storage.test.ts diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 9fa8ba49f..c22db1d00 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -262,25 +262,26 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor jwt_exp, id ) - SELECT + VALUES ${{ type: 'varchar', value: user_id }}, ${{ type: 'varchar', value: client_id }}, ${{ type: 1184, value: connectIsoString }}, ${{ type: 'varchar', value: sdk }}, ${{ type: 'varchar', value: user_agent }}, ${{ type: 1184, value: jwtExpIsoString }}, - ${{ type: 'varchar', value: uuid }} + ${{ type: 'varchar', value: uuid }} / / WHERE - NOT EXISTS ( + / / NOT EXISTS ( + / / SELECT - 1 + / / 1 / / FROM - sdk_report_events + / / sdk_report_events / / WHERE - user_id = ${{ type: 'varchar', value: user_id }} - AND client_id = ${{ type: 'varchar', value: client_id }} - AND connect_at >= ${{ type: 1184, value: gte }} - AND connect_at < ${{ type: 1184, value: lt }} + / / user_id = ${{ type: 'varchar', value: user_id }} / / + AND client_id = ${{ type: 'varchar', value: client_id }} / / + AND connect_at >= ${{ type: 1184, value: gte }} / / + AND connect_at < ${{ type: 1184, value: lt }} / / ); `.execute(); } diff --git a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts index 1e6015cd6..37f687200 100644 --- a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts +++ b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts @@ -2,6 +2,7 @@ import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOption import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'; import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js'; import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; +import { PostgresReportStorageFactory } from './PostgresReportStorageFactory.js'; export type PostgresTestStorageOptions = { url: string; @@ -48,6 +49,21 @@ export const postgresTestSetup = (factoryOptions: PostgresTestStorageOptions) => }; return { + reportFactory: async (options?: TestStorageOptions) => { + try { + if (!options?.doNotClear) { + await migrate(framework.migrations.Direction.Up); + } + + return new PostgresReportStorageFactory({ + config: TEST_CONNECTION_OPTIONS + }); + } catch (ex) { + // Vitest does not display these errors nicely when using the `await using` syntx + console.error(ex, ex.cause); + throw ex; + } + }, factory: async (options?: TestStorageOptions) => { try { if (!options?.doNotClear) { diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts new file mode 100644 index 000000000..ef6fd5b51 --- /dev/null +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -0,0 +1,11 @@ +import { beforeAll, describe, it } from 'vitest'; +import { POSTGRES_REPORT_STORAGE_FACTORY } from './util.js'; + +describe('SDK reporting storage', () => { + beforeAll(async () => { + const { db } = await POSTGRES_REPORT_STORAGE_FACTORY(); + }); + it('Should have tables declared', async () => { + const { db } = await POSTGRES_REPORT_STORAGE_FACTORY(); + }); +}); diff --git a/modules/module-postgres-storage/test/src/util.ts b/modules/module-postgres-storage/test/src/util.ts index c9e0e2555..19b81106e 100644 --- a/modules/module-postgres-storage/test/src/util.ts +++ b/modules/module-postgres-storage/test/src/util.ts @@ -2,10 +2,7 @@ import path from 'path'; import { fileURLToPath } from 'url'; import { normalizePostgresStorageConfig } from '../../src//types/types.js'; import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js'; -import { - postgresTestSetup, - PostgresTestStorageFactoryGenerator -} from '../../src/storage/PostgresTestStorageFactoryGenerator.js'; +import { postgresTestSetup } from '../../src/storage/PostgresTestStorageFactoryGenerator.js'; import { env } from './env.js'; const __filename = fileURLToPath(import.meta.url); @@ -37,3 +34,4 @@ export const POSTGRES_STORAGE_SETUP = postgresTestSetup({ }); export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.factory; +export const POSTGRES_REPORT_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.reportFactory; From 4a930e99d753823560b17a66e2608b6d204d3128 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 17:05:14 +0200 Subject: [PATCH 132/169] removed query --- .../src/storage/PostgresReportStorageFactory.ts | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index c22db1d00..471ae8b19 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -269,20 +269,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor ${{ type: 'varchar', value: sdk }}, ${{ type: 'varchar', value: user_agent }}, ${{ type: 1184, value: jwtExpIsoString }}, - ${{ type: 'varchar', value: uuid }} / / - WHERE - / / NOT EXISTS ( - / / - SELECT - / / 1 / / - FROM - / / sdk_report_events / / - WHERE - / / user_id = ${{ type: 'varchar', value: user_id }} / / - AND client_id = ${{ type: 'varchar', value: client_id }} / / - AND connect_at >= ${{ type: 1184, value: gte }} / / - AND connect_at < ${{ type: 1184, value: lt }} / / - ); + ${{ type: 'varchar', value: uuid }} `.execute(); } } From ac40bcac3cb9f7f04dd477f6d4d0337be93d6328 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 17:14:08 +0200 Subject: [PATCH 133/169] removed query --- .../src/storage/PostgresReportStorageFactory.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 471ae8b19..85746978c 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -263,13 +263,15 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor id ) VALUES - ${{ type: 'varchar', value: user_id }}, - ${{ type: 'varchar', value: client_id }}, - ${{ type: 1184, value: connectIsoString }}, - ${{ type: 'varchar', value: sdk }}, - ${{ type: 'varchar', value: user_agent }}, - ${{ type: 1184, value: jwtExpIsoString }}, - ${{ type: 'varchar', value: uuid }} + ( + ${{ type: 'varchar', value: user_id }}, + ${{ type: 'varchar', value: client_id }}, + ${{ type: 1184, value: connectIsoString }}, + ${{ type: 'varchar', value: sdk }}, + ${{ type: 'varchar', value: user_agent }}, + ${{ type: 1184, value: jwtExpIsoString }}, + ${{ type: 'varchar', value: uuid }} + ) `.execute(); } } From 5020173fff1d3e8a5b5843ff8ae473709bcedf40 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 17:24:03 +0200 Subject: [PATCH 134/169] trying to simplify the update --- .../storage/PostgresReportStorageFactory.ts | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 85746978c..cefb79d0a 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -249,31 +249,32 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor AND client_id = ${{ type: 'varchar', value: client_id }} AND connect_at >= ${{ type: 1184, value: gte }} AND connect_at < ${{ type: 1184, value: lt }}; - `.rows(); - if (result.length === 0) { - await this.db.sql` - INSERT INTO - sdk_report_events ( - user_id, - client_id, - connect_at, - sdk, - user_agent, - jwt_exp, - id - ) - VALUES - ( - ${{ type: 'varchar', value: user_id }}, - ${{ type: 'varchar', value: client_id }}, - ${{ type: 1184, value: connectIsoString }}, - ${{ type: 'varchar', value: sdk }}, - ${{ type: 'varchar', value: user_agent }}, - ${{ type: 1184, value: jwtExpIsoString }}, - ${{ type: 'varchar', value: uuid }} - ) - `.execute(); - } + `.execute(); + console.log(result.results); + // if (result.results[] === 0) { + // await this.db.sql` + // INSERT INTO + // sdk_report_events ( + // user_id, + // client_id, + // connect_at, + // sdk, + // user_agent, + // jwt_exp, + // id + // ) + // VALUES + // ( + // ${{ type: 'varchar', value: user_id }}, + // ${{ type: 'varchar', value: client_id }}, + // ${{ type: 1184, value: connectIsoString }}, + // ${{ type: 'varchar', value: sdk }}, + // ${{ type: 'varchar', value: user_agent }}, + // ${{ type: 1184, value: jwtExpIsoString }}, + // ${{ type: 'varchar', value: uuid }} + // ) + // `.execute(); + // } } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { const { user_id, client_id, disconnect_at } = data; From c2b84d27d3fe2f52019d407faf5b57dd912eec03 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 30 Jul 2025 17:31:38 +0200 Subject: [PATCH 135/169] trying to simplify the update --- .../storage/PostgresReportStorageFactory.ts | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index cefb79d0a..d888f0f9b 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -250,31 +250,30 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor AND connect_at >= ${{ type: 1184, value: gte }} AND connect_at < ${{ type: 1184, value: lt }}; `.execute(); - console.log(result.results); - // if (result.results[] === 0) { - // await this.db.sql` - // INSERT INTO - // sdk_report_events ( - // user_id, - // client_id, - // connect_at, - // sdk, - // user_agent, - // jwt_exp, - // id - // ) - // VALUES - // ( - // ${{ type: 'varchar', value: user_id }}, - // ${{ type: 'varchar', value: client_id }}, - // ${{ type: 1184, value: connectIsoString }}, - // ${{ type: 'varchar', value: sdk }}, - // ${{ type: 'varchar', value: user_agent }}, - // ${{ type: 1184, value: jwtExpIsoString }}, - // ${{ type: 'varchar', value: uuid }} - // ) - // `.execute(); - // } + if (result.results[1].status === 'UPDATE 0') { + await this.db.sql` + INSERT INTO + sdk_report_events ( + user_id, + client_id, + connect_at, + sdk, + user_agent, + jwt_exp, + id + ) + VALUES + ( + ${{ type: 'varchar', value: user_id }}, + ${{ type: 'varchar', value: client_id }}, + ${{ type: 1184, value: connectIsoString }}, + ${{ type: 'varchar', value: sdk }}, + ${{ type: 'varchar', value: user_agent }}, + ${{ type: 1184, value: jwtExpIsoString }}, + ${{ type: 'varchar', value: uuid }} + ) + `.execute(); + } } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { const { user_id, client_id, disconnect_at } = data; From fc82f1491dbdbc08ecbbded42681c888813b03ba Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 31 Jul 2025 11:19:26 +0200 Subject: [PATCH 136/169] tests tests --- .../sdk-report-storage.test.ts.snap | 27 ++++ .../test/src/sdk-report-storage.test.ts | 145 +++++++++++++++++- 2 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap diff --git a/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap new file mode 100644 index 000000000..797ce75b9 --- /dev/null +++ b/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap @@ -0,0 +1,27 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`SDK reporting storage > Should show currently connected users with start range 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + ], + "users": 1, +} +`; + +exports[`SDK reporting storage > Should show currently connected users with start range and end range 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-js/1.21.0", + "users": 1, + }, + ], + "users": 1, +} +`; diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index ef6fd5b51..2cf16021a 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -1,11 +1,152 @@ -import { beforeAll, describe, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { POSTGRES_REPORT_STORAGE_FACTORY } from './util.js'; describe('SDK reporting storage', () => { + const now = new Date(); + const nowAdd5minutes = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() + 5 + ); + const nowLess5minutes = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() - 5 + ); + const yesterday = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1); + const user_one = { + user_id: 'user_one', + client_id: 'client_one', + connect_at: now.toISOString(), + sdk: 'powersync-dart/1.6.4', + user_agent: 'powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android', + jwt_exp: nowAdd5minutes.toISOString(), + id: '1' + }; + const user_two = { + user_id: 'user_two', + client_id: 'client_two', + connect_at: nowLess5minutes.toISOString(), + sdk: 'powersync-js/1.21.0', + user_agent: 'powersync-js/1.21.0 powersync-web Chromium/138 linux', + jwt_exp: nowAdd5minutes.toISOString(), + id: '2' + }; + const user_three = { + user_id: 'user_three', + client_id: 'client_three', + connect_at: yesterday.toISOString(), + sdk: 'powersync-js/1.21.0', + user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', + disconnect_at: yesterday.toISOString(), + id: '3' + }; + + const user_four = { + user_id: 'user_four', + client_id: 'user_five', + connect_at: now.toISOString(), + sdk: 'powersync-js/1.21.0', + user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', + jwt_exp: nowLess5minutes.toISOString(), + id: '4' + }; + beforeAll(async () => { const { db } = await POSTGRES_REPORT_STORAGE_FACTORY(); + const result = await db.sql` + INSERT INTO + sdk_report_events ( + user_id, + client_id, + connect_at, + sdk, + user_agent, + jwt_exp, + id, + disconnect_at + ) + VALUES + ( + ${{ type: 'varchar', value: user_one.user_id }}, + ${{ type: 'varchar', value: user_one.client_id }}, + ${{ type: 1184, value: user_one.connect_at }}, + ${{ type: 'varchar', value: user_one.sdk }}, + ${{ type: 'varchar', value: user_one.user_agent }}, + ${{ type: 1184, value: user_one.jwt_exp }}, + ${{ type: 'varchar', value: user_one.id }}, + NULL + ), + ( + ${{ type: 'varchar', value: user_two.user_id }}, + ${{ type: 'varchar', value: user_two.client_id }}, + ${{ type: 1184, value: user_two.connect_at }}, + ${{ type: 'varchar', value: user_two.sdk }}, + ${{ type: 'varchar', value: user_two.user_agent }}, + ${{ type: 1184, value: user_two.jwt_exp }}, + ${{ type: 'varchar', value: user_two.id }}, + NULL + ), + ( + ${{ type: 'varchar', value: user_four.user_id }}, + ${{ type: 'varchar', value: user_four.client_id }}, + ${{ type: 1184, value: user_four.connect_at }}, + ${{ type: 'varchar', value: user_four.sdk }}, + ${{ type: 'varchar', value: user_four.user_agent }}, + ${{ type: 1184, value: user_four.jwt_exp }}, + ${{ type: 'varchar', value: user_four.id }}, + NULL + ), + ( + ${{ type: 'varchar', value: user_three.user_id }}, + ${{ type: 'varchar', value: user_three.client_id }}, + ${{ type: 1184, value: user_three.connect_at }}, + ${{ type: 'varchar', value: user_three.sdk }}, + ${{ type: 'varchar', value: user_three.user_agent }}, + NULL, + ${{ type: 'varchar', value: user_three.id }}, + ${{ type: 1184, value: user_three.disconnect_at }} + ) + `.execute(); + console.log(result); }); - it('Should have tables declared', async () => { + afterAll(async () => { const { db } = await POSTGRES_REPORT_STORAGE_FACTORY(); + await db.sql`DELETE FROM sdk_report_events`.execute(); + }); + it('Should show currently connected users with start range', async () => { + const factory = await POSTGRES_REPORT_STORAGE_FACTORY(); + const current = await factory.listCurrentConnections({ + range: { + start_date: new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() - 1 + ).toISOString() + } + }); + expect(current).toMatchSnapshot(); + }); + it('Should show currently connected users with start range and end range', async () => { + const factory = await POSTGRES_REPORT_STORAGE_FACTORY(); + const current = await factory.listCurrentConnections({ + range: { + end_date: nowLess5minutes.toISOString(), + start_date: new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() - 6 + ).toISOString() + } + }); + expect(current).toMatchSnapshot(); }); }); From a9d453f1bdc069d2b47f610323f96231e5b3477d Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 31 Jul 2025 11:49:59 +0200 Subject: [PATCH 137/169] postgres tests WIP --- modules/module-postgres-storage/package.json | 2 +- .../sdk-report-storage.test.ts.snap | 86 ++++++++++++++++++- .../test/src/sdk-report-storage.test.ts | 77 +++++++++++++++-- 3 files changed, 154 insertions(+), 11 deletions(-) diff --git a/modules/module-postgres-storage/package.json b/modules/module-postgres-storage/package.json index 2336d09c1..0cefe7e29 100644 --- a/modules/module-postgres-storage/package.json +++ b/modules/module-postgres-storage/package.json @@ -12,7 +12,7 @@ "build": "tsc -b", "build:tests": "tsc -b test/tsconfig.json", "clean": "rm -rf ./lib && tsc -b --clean", - "test": "vitest" + "test": "vitest --poolOptions.threads.singleThread" }, "exports": { ".": { diff --git a/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap index 797ce75b9..44226c031 100644 --- a/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap +++ b/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap @@ -1,6 +1,88 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html -exports[`SDK reporting storage > Should show currently connected users with start range 1`] = ` +exports[`SDK reporting storage > Should show SDK scrape data for user over the past day 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 2, + "sdk": "powersync-js/1.21.0", + "users": 2, + }, + ], + "users": 3, +} +`; + +exports[`SDK reporting storage > Should show SDK scrape data for user over the past month 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 3, + "sdk": "powersync-js/1.21.0", + "users": 3, + }, + { + "clients": 1, + "sdk": "powersync-js/1.23.0", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.24.0", + "users": 1, + }, + ], + "users": 6, +} +`; + +exports[`SDK reporting storage > Should show SDK scrape data for user over the past week 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 3, + "sdk": "powersync-js/1.21.0", + "users": 3, + }, + ], + "users": 4, +} +`; + +exports[`SDK reporting storage > Should show all currently connected users 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.0", + "users": 1, + }, + ], + "users": 2, +} +`; + +exports[`SDK reporting storage > Should show connected users with start range 1`] = ` { "sdks": [ { @@ -13,7 +95,7 @@ exports[`SDK reporting storage > Should show currently connected users with star } `; -exports[`SDK reporting storage > Should show currently connected users with start range and end range 1`] = ` +exports[`SDK reporting storage > Should show connected users with start range and end range 1`] = ` { "sdks": [ { diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index 2cf16021a..517005c49 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -1,7 +1,8 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { POSTGRES_REPORT_STORAGE_FACTORY } from './util.js'; -describe('SDK reporting storage', () => { +describe('SDK reporting storage', async () => { + const factory = await POSTGRES_REPORT_STORAGE_FACTORY(); const now = new Date(); const nowAdd5minutes = new Date( now.getFullYear(), @@ -18,6 +19,8 @@ describe('SDK reporting storage', () => { now.getMinutes() - 5 ); const yesterday = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1); + const weekAgo = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7); + const monthAgo = new Date(now.getFullYear(), now.getMonth() - 1, now.getDate()); const user_one = { user_id: 'user_one', client_id: 'client_one', @@ -48,7 +51,7 @@ describe('SDK reporting storage', () => { const user_four = { user_id: 'user_four', - client_id: 'user_five', + client_id: 'client_four', connect_at: now.toISOString(), sdk: 'powersync-js/1.21.0', user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', @@ -56,9 +59,28 @@ describe('SDK reporting storage', () => { id: '4' }; + const user_week = { + user_id: 'user_week', + client_id: 'client_week', + connect_at: weekAgo.toISOString(), + sdk: 'powersync-js/1.24.0', + user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', + disconnect_at: weekAgo.toISOString(), + id: 'week' + }; + + const user_month = { + user_id: 'user_month', + client_id: 'client_month', + connect_at: monthAgo.toISOString(), + sdk: 'powersync-js/1.23.0', + user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', + disconnect_at: monthAgo.toISOString(), + id: 'month' + }; + beforeAll(async () => { - const { db } = await POSTGRES_REPORT_STORAGE_FACTORY(); - const result = await db.sql` + const result = await factory.db.sql` INSERT INTO sdk_report_events ( user_id, @@ -110,6 +132,26 @@ describe('SDK reporting storage', () => { NULL, ${{ type: 'varchar', value: user_three.id }}, ${{ type: 1184, value: user_three.disconnect_at }} + ), + ( + ${{ type: 'varchar', value: user_week.user_id }}, + ${{ type: 'varchar', value: user_week.client_id }}, + ${{ type: 1184, value: user_week.connect_at }}, + ${{ type: 'varchar', value: user_week.sdk }}, + ${{ type: 'varchar', value: user_week.user_agent }}, + NULL, + ${{ type: 'varchar', value: user_week.id }}, + ${{ type: 1184, value: user_week.disconnect_at }} + ), + ( + ${{ type: 'varchar', value: user_month.user_id }}, + ${{ type: 'varchar', value: user_month.client_id }}, + ${{ type: 1184, value: user_month.connect_at }}, + ${{ type: 'varchar', value: user_month.sdk }}, + ${{ type: 'varchar', value: user_month.user_agent }}, + NULL, + ${{ type: 'varchar', value: user_month.id }}, + ${{ type: 1184, value: user_month.disconnect_at }} ) `.execute(); console.log(result); @@ -118,8 +160,7 @@ describe('SDK reporting storage', () => { const { db } = await POSTGRES_REPORT_STORAGE_FACTORY(); await db.sql`DELETE FROM sdk_report_events`.execute(); }); - it('Should show currently connected users with start range', async () => { - const factory = await POSTGRES_REPORT_STORAGE_FACTORY(); + it('Should show connected users with start range', async () => { const current = await factory.listCurrentConnections({ range: { start_date: new Date( @@ -133,8 +174,7 @@ describe('SDK reporting storage', () => { }); expect(current).toMatchSnapshot(); }); - it('Should show currently connected users with start range and end range', async () => { - const factory = await POSTGRES_REPORT_STORAGE_FACTORY(); + it('Should show connected users with start range and end range', async () => { const current = await factory.listCurrentConnections({ range: { end_date: nowLess5minutes.toISOString(), @@ -149,4 +189,25 @@ describe('SDK reporting storage', () => { }); expect(current).toMatchSnapshot(); }); + it('Should show SDK scrape data for user over the past month', async () => { + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'month' + }); + expect(sdk).toMatchSnapshot(); + }); + it('Should show SDK scrape data for user over the past week', async () => { + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'week' + }); + expect(sdk).toMatchSnapshot(); + }); + it('Should show SDK scrape data for user over the past day', async () => { + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'day' + }); + expect(sdk).toMatchSnapshot(); + }); }); From 7baf97b9007fb678bcf78c76a94fffef6d459b86 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 31 Jul 2025 11:50:44 +0200 Subject: [PATCH 138/169] removed command --- modules/module-postgres-storage/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-postgres-storage/package.json b/modules/module-postgres-storage/package.json index 0cefe7e29..2336d09c1 100644 --- a/modules/module-postgres-storage/package.json +++ b/modules/module-postgres-storage/package.json @@ -12,7 +12,7 @@ "build": "tsc -b", "build:tests": "tsc -b test/tsconfig.json", "clean": "rm -rf ./lib && tsc -b --clean", - "test": "vitest --poolOptions.threads.singleThread" + "test": "vitest" }, "exports": { ".": { From 65120ce22dfd49090c5e07228f1d0d2d35b1e832 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 31 Jul 2025 12:43:02 +0200 Subject: [PATCH 139/169] application name conflict report storage --- .../src/storage/PostgresReportStorageFactory.ts | 7 +++---- .../test/src/sdk-report-storage.test.ts | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index d888f0f9b..f4339fcc1 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -1,4 +1,4 @@ -import { storage } from '@powersync/service-core'; +import { POWERSYNC_VERSION, storage } from '@powersync/service-core'; import * as pg_wire from '@powersync/service-jpgwire'; import { event_types } from '@powersync/service-types'; import { v4 } from 'uuid'; @@ -6,7 +6,6 @@ import * as lib_postgres from '@powersync/lib-service-postgres'; import { NormalizedPostgresStorageConfig } from '../types/types.js'; import { STORAGE_SCHEMA_NAME } from '../utils/db.js'; -import { getStorageApplicationName } from '../utils/application-name.js'; import { DeleteOldSdkData, ListCurrentConnections, @@ -29,7 +28,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor this.db = new lib_postgres.DatabaseClient({ config: options.config, schema: STORAGE_SCHEMA_NAME, - applicationName: getStorageApplicationName() + applicationName: `powersync-report-storage/${POWERSYNC_VERSION}` }); this.db.registerListener({ @@ -368,7 +367,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } async [Symbol.asyncDispose]() { - await this.db[Symbol.asyncDispose](); + // await this.db[Symbol.asyncDispose](); } async prepareStatements(connection: pg_wire.PgConnection) { diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index 517005c49..c905e6159 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -157,8 +157,7 @@ describe('SDK reporting storage', async () => { console.log(result); }); afterAll(async () => { - const { db } = await POSTGRES_REPORT_STORAGE_FACTORY(); - await db.sql`DELETE FROM sdk_report_events`.execute(); + await factory.db.sql`DELETE FROM sdk_report_events`.execute(); }); it('Should show connected users with start range', async () => { const current = await factory.listCurrentConnections({ From da94778fb9c765ec7c46ad9df95fcb4052fb06f1 Mon Sep 17 00:00:00 2001 From: JuanB Date: Fri, 1 Aug 2025 00:01:46 +0200 Subject: [PATCH 140/169] checking concurrency issues in tests --- modules/module-postgres-storage/package.json | 2 +- .../migrations/scripts/1684951997326-init.ts | 20 +++++++++ .../scripts/1753871047719-sdk_report.ts | 45 ------------------- .../storage/PostgresReportStorageFactory.ts | 8 ++-- .../PostgresTestStorageFactoryGenerator.ts | 6 +-- .../module-postgres-storage/src/utils/db.ts | 1 + .../test/src/sdk-report-storage.test.ts | 5 +-- 7 files changed, 30 insertions(+), 57 deletions(-) delete mode 100644 modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts diff --git a/modules/module-postgres-storage/package.json b/modules/module-postgres-storage/package.json index 2336d09c1..0cefe7e29 100644 --- a/modules/module-postgres-storage/package.json +++ b/modules/module-postgres-storage/package.json @@ -12,7 +12,7 @@ "build": "tsc -b", "build:tests": "tsc -b test/tsconfig.json", "clean": "rm -rf ./lib && tsc -b --clean", - "test": "vitest" + "test": "vitest --poolOptions.threads.singleThread" }, "exports": { ".": { diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index 61e8699ea..d54669d11 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -128,6 +128,26 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { CONSTRAINT unique_user_sync PRIMARY KEY (user_id, sync_rules_id) ); `.execute(); + await db.sql` + CREATE TABLE sdk_report_events ( + id TEXT PRIMARY KEY, + user_agent TEXT NOT NULL, + client_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sdk TEXT NOT NULL, + jwt_exp TIMESTAMP WITH TIME ZONE, + connect_at TIMESTAMP WITH TIME ZONE NOT NULL, + disconnect_at TIMESTAMP WITH TIME ZONE + ) + `.execute(); + + await db.sql` CREATE INDEX sdk_list_index ON sdk_report_events (connect_at, jwt_exp, disconnect_at) `.execute(); + + await db.sql`CREATE INDEX sdk_user_id_index ON sdk_report_events (user_id)`.execute(); + + await db.sql`CREATE INDEX sdk_client_id_index ON sdk_report_events (client_id)`.execute(); + + await db.sql`CREATE INDEX sdk_index ON sdk_report_events (sdk)`.execute(); }); }; diff --git a/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts b/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts deleted file mode 100644 index c19f2f921..000000000 --- a/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { migrations } from '@powersync/service-core'; - -import { openMigrationDB } from '../migration-utils.js'; - -export const up: migrations.PowerSyncMigrationFunction = async (context) => { - const { - service_context: { configuration } - } = context; - await using client = openMigrationDB(configuration.storage); - - await client.transaction(async (db) => { - await db.sql` - CREATE TABLE IF NOT EXISTS sdk_report_events ( - id TEXT PRIMARY KEY, - user_agent TEXT NOT NULL, - client_id TEXT NOT NULL, - user_id TEXT NOT NULL, - sdk TEXT NOT NULL, - jwt_exp TIMESTAMP WITH TIME ZONE, - connect_at TIMESTAMP WITH TIME ZONE NOT NULL, - disconnect_at TIMESTAMP WITH TIME ZONE - ) - `.execute(); - - await db.sql` - CREATE INDEX IF NOT EXISTS sdk_list_index ON sdk_report_events (connect_at, jwt_exp, disconnect_at) - `.execute(); - - await db.sql`CREATE INDEX IF NOT EXISTS sdk_user_id_index ON sdk_report_events (user_id)`.execute(); - - await db.sql`CREATE INDEX IF NOT EXISTS sdk_client_id_index ON sdk_report_events (client_id)`.execute(); - - await db.sql`CREATE INDEX IF NOT EXISTS sdk_index ON sdk_report_events (sdk)`.execute(); - }); -}; - -export const down: migrations.PowerSyncMigrationFunction = async (context) => { - const { - service_context: { configuration } - } = context; - await using client = openMigrationDB(configuration.storage); - client.lockConnection(async (db) => { - await db.sql`DROP TABLE IF EXISTS sdk_report_events`.execute(); - }); -}; diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index f4339fcc1..5cbbd9a5c 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -1,11 +1,9 @@ -import { POWERSYNC_VERSION, storage } from '@powersync/service-core'; +import { storage } from '@powersync/service-core'; import * as pg_wire from '@powersync/service-jpgwire'; import { event_types } from '@powersync/service-types'; import { v4 } from 'uuid'; import * as lib_postgres from '@powersync/lib-service-postgres'; import { NormalizedPostgresStorageConfig } from '../types/types.js'; - -import { STORAGE_SCHEMA_NAME } from '../utils/db.js'; import { DeleteOldSdkData, ListCurrentConnections, @@ -17,6 +15,8 @@ import { import { SdkReporting, SdkReportingDecoded } from '../types/models/SdkReporting.js'; import { toInteger } from 'ix/util/tointeger.js'; import { logger } from '@powersync/lib-services-framework'; +import { getStorageApplicationName } from '../utils/application-name.js'; +import { STORAGE_SCHEMA_NAME } from '../utils/db.js'; export type PostgresReportStorageOptions = { config: NormalizedPostgresStorageConfig; @@ -28,7 +28,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor this.db = new lib_postgres.DatabaseClient({ config: options.config, schema: STORAGE_SCHEMA_NAME, - applicationName: `powersync-report-storage/${POWERSYNC_VERSION}` + applicationName: getStorageApplicationName() }); this.db.registerListener({ diff --git a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts index 37f687200..47d4922d6 100644 --- a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts +++ b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts @@ -51,9 +51,9 @@ export const postgresTestSetup = (factoryOptions: PostgresTestStorageOptions) => return { reportFactory: async (options?: TestStorageOptions) => { try { - if (!options?.doNotClear) { - await migrate(framework.migrations.Direction.Up); - } + // if (!options?.doNotClear) { + // await migrate(framework.migrations.Direction.Up); + // } return new PostgresReportStorageFactory({ config: TEST_CONNECTION_OPTIONS diff --git a/modules/module-postgres-storage/src/utils/db.ts b/modules/module-postgres-storage/src/utils/db.ts index 500cb3aa8..11ded913a 100644 --- a/modules/module-postgres-storage/src/utils/db.ts +++ b/modules/module-postgres-storage/src/utils/db.ts @@ -23,5 +23,6 @@ export const dropTables = async (client: lib_postgres.DatabaseClient) => { await db.sql`DROP TABLE IF EXISTS custom_write_checkpoints`.execute(); await db.sql`DROP SEQUENCE IF EXISTS op_id_sequence`.execute(); await db.sql`DROP SEQUENCE IF EXISTS sync_rules_id_sequence`.execute(); + await db.sql`DROP TABLE IF EXISTS sdk_report_events`.execute(); }); }; diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index c905e6159..4e88220f8 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { beforeAll, describe, expect, it } from 'vitest'; import { POSTGRES_REPORT_STORAGE_FACTORY } from './util.js'; describe('SDK reporting storage', async () => { @@ -156,9 +156,6 @@ describe('SDK reporting storage', async () => { `.execute(); console.log(result); }); - afterAll(async () => { - await factory.db.sql`DELETE FROM sdk_report_events`.execute(); - }); it('Should show connected users with start range', async () => { const current = await factory.listCurrentConnections({ range: { From de57ca02e2d0e19ca8f4b55fd2e23f75f1341165 Mon Sep 17 00:00:00 2001 From: JuanB Date: Fri, 1 Aug 2025 00:27:25 +0200 Subject: [PATCH 141/169] test chnages to tests --- modules/module-postgres-storage/package.json | 2 +- .../src/storage/PostgresReportStorageFactory.ts | 2 +- .../src/storage/PostgresTestStorageFactoryGenerator.ts | 6 +++--- .../test/src/sdk-report-storage.test.ts | 1 - 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/modules/module-postgres-storage/package.json b/modules/module-postgres-storage/package.json index 0cefe7e29..2336d09c1 100644 --- a/modules/module-postgres-storage/package.json +++ b/modules/module-postgres-storage/package.json @@ -12,7 +12,7 @@ "build": "tsc -b", "build:tests": "tsc -b test/tsconfig.json", "clean": "rm -rf ./lib && tsc -b --clean", - "test": "vitest --poolOptions.threads.singleThread" + "test": "vitest" }, "exports": { ".": { diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 5cbbd9a5c..5df0fb493 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -54,7 +54,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const parsedIsoString = parsedDate.toISOString(); switch (timeframe) { case 'month': { - return { lt: parsedIsoString, gt: new Date(year, parsedDate.getMonth() - interval).toISOString() }; + return { lt: parsedIsoString, gt: new Date(year, parsedDate.getMonth() - interval - 1).toISOString() }; } case 'week': { const weekStartDate = new Date(parsedDate); diff --git a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts index 47d4922d6..37f687200 100644 --- a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts +++ b/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts @@ -51,9 +51,9 @@ export const postgresTestSetup = (factoryOptions: PostgresTestStorageOptions) => return { reportFactory: async (options?: TestStorageOptions) => { try { - // if (!options?.doNotClear) { - // await migrate(framework.migrations.Direction.Up); - // } + if (!options?.doNotClear) { + await migrate(framework.migrations.Direction.Up); + } return new PostgresReportStorageFactory({ config: TEST_CONNECTION_OPTIONS diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index 4e88220f8..66cc6c69d 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -154,7 +154,6 @@ describe('SDK reporting storage', async () => { ${{ type: 1184, value: user_month.disconnect_at }} ) `.execute(); - console.log(result); }); it('Should show connected users with start range', async () => { const current = await factory.listCurrentConnections({ From 05006be801aaf7e4850c717f531f684d8d861885 Mon Sep 17 00:00:00 2001 From: JuanB Date: Fri, 1 Aug 2025 13:02:00 +0200 Subject: [PATCH 142/169] completed postgres tests --- .../storage/PostgresReportStorageFactory.ts | 4 +- .../sdk-report-storage.test.ts.snap | 125 ++++++++++++--- .../test/src/sdk-report-storage.test.ts | 146 +++++++++++++++++- .../module-postgres-storage/test/src/util.ts | 4 +- packages/types/src/events.ts | 1 + 5 files changed, 244 insertions(+), 36 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 5df0fb493..ff589bbc1 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -82,8 +82,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } } - private timeFrameDeleteQuery(timeframe: event_types.TimeFrames, interval: number = 1) { - const { year, month, today, parsedDate } = this.parseJsDate(new Date()); + private timeFrameDeleteQuery(timeframe: event_types.TimeFrames, interval: number = 1, test_date?: Date) { + const { year, month, today, parsedDate } = test_date ? this.parseJsDate(test_date) : this.parseJsDate(new Date()); switch (timeframe) { case 'month': { return { lt: new Date(year, parsedDate.getMonth() - interval).toISOString() }; diff --git a/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap index 44226c031..7ddd94b2d 100644 --- a/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap +++ b/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap @@ -1,6 +1,23 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html -exports[`SDK reporting storage > Should show SDK scrape data for user over the past day 1`] = ` +exports[`SDK reporting storage > Should create a sdk event if its after a day 1`] = ` +[ + { + "client_id": "client_week", + "sdk": "powersync-js/1.24.5", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_week", + }, + { + "client_id": "client_week", + "sdk": "powersync-js/1.24.5", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_week", + }, +] +`; + +exports[`SDK reporting storage > Should delete rows older than specified range 1`] = ` { "sdks": [ { @@ -9,16 +26,26 @@ exports[`SDK reporting storage > Should show SDK scrape data for user over the p "users": 1, }, { - "clients": 2, - "sdk": "powersync-js/1.21.0", - "users": 2, + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, }, ], - "users": 3, + "users": 4, } `; -exports[`SDK reporting storage > Should show SDK scrape data for user over the past month 1`] = ` +exports[`SDK reporting storage > Should show SDK scrape data for user over the past day 1`] = ` { "sdks": [ { @@ -26,27 +53,22 @@ exports[`SDK reporting storage > Should show SDK scrape data for user over the p "sdk": "powersync-dart/1.6.4", "users": 1, }, - { - "clients": 3, - "sdk": "powersync-js/1.21.0", - "users": 3, - }, { "clients": 1, - "sdk": "powersync-js/1.23.0", + "sdk": "powersync-js/1.21.1", "users": 1, }, { "clients": 1, - "sdk": "powersync-js/1.24.0", + "sdk": "powersync-js/1.21.4", "users": 1, }, ], - "users": 6, + "users": 3, } `; -exports[`SDK reporting storage > Should show SDK scrape data for user over the past week 1`] = ` +exports[`SDK reporting storage > Should show SDK scrape data for user over the past month 1`] = ` { "sdks": [ { @@ -55,16 +77,41 @@ exports[`SDK reporting storage > Should show SDK scrape data for user over the p "users": 1, }, { - "clients": 3, - "sdk": "powersync-js/1.21.0", - "users": 3, + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.23.6", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.23.7", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, }, ], - "users": 4, + "users": 7, } `; -exports[`SDK reporting storage > Should show all currently connected users 1`] = ` +exports[`SDK reporting storage > Should show SDK scrape data for user over the past week 1`] = ` { "sdks": [ { @@ -74,11 +121,21 @@ exports[`SDK reporting storage > Should show all currently connected users 1`] = }, { "clients": 1, - "sdk": "powersync-js/1.21.0", + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", "users": 1, }, ], - "users": 2, + "users": 4, } `; @@ -100,10 +157,32 @@ exports[`SDK reporting storage > Should show connected users with start range an "sdks": [ { "clients": 1, - "sdk": "powersync-js/1.21.0", + "sdk": "powersync-js/1.21.1", "users": 1, }, ], "users": 1, } `; + +exports[`SDK reporting storage > Should update a connected sdk event and make it disconnected 1`] = ` +[ + { + "client_id": "client_one", + "sdk": "powersync-dart/1.6.4", + "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android", + "user_id": "user_one", + }, +] +`; + +exports[`SDK reporting storage > Should update a sdk event if its within a day 1`] = ` +[ + { + "client_id": "client_one", + "sdk": "powersync-dart/1.6.4", + "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android", + "user_id": "user_one", + }, +] +`; diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index 66cc6c69d..a5666f277 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -1,5 +1,15 @@ -import { beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { POSTGRES_REPORT_STORAGE_FACTORY } from './util.js'; +import { event_types } from '@powersync/service-types'; + +function removeVolatileFields(sdks: event_types.SdkConnectDocument[]): Partial[] { + return sdks.map((sdk) => { + const { id, disconnect_at, connect_at, jwt_exp, ...rest } = sdk; + return { + ...rest + }; + }); +} describe('SDK reporting storage', async () => { const factory = await POSTGRES_REPORT_STORAGE_FACTORY(); @@ -34,7 +44,7 @@ describe('SDK reporting storage', async () => { user_id: 'user_two', client_id: 'client_two', connect_at: nowLess5minutes.toISOString(), - sdk: 'powersync-js/1.21.0', + sdk: 'powersync-js/1.21.1', user_agent: 'powersync-js/1.21.0 powersync-web Chromium/138 linux', jwt_exp: nowAdd5minutes.toISOString(), id: '2' @@ -43,7 +53,7 @@ describe('SDK reporting storage', async () => { user_id: 'user_three', client_id: 'client_three', connect_at: yesterday.toISOString(), - sdk: 'powersync-js/1.21.0', + sdk: 'powersync-js/1.21.2', user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', disconnect_at: yesterday.toISOString(), id: '3' @@ -53,7 +63,7 @@ describe('SDK reporting storage', async () => { user_id: 'user_four', client_id: 'client_four', connect_at: now.toISOString(), - sdk: 'powersync-js/1.21.0', + sdk: 'powersync-js/1.21.4', user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', jwt_exp: nowLess5minutes.toISOString(), id: '4' @@ -63,7 +73,7 @@ describe('SDK reporting storage', async () => { user_id: 'user_week', client_id: 'client_week', connect_at: weekAgo.toISOString(), - sdk: 'powersync-js/1.24.0', + sdk: 'powersync-js/1.24.5', user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', disconnect_at: weekAgo.toISOString(), id: 'week' @@ -73,14 +83,24 @@ describe('SDK reporting storage', async () => { user_id: 'user_month', client_id: 'client_month', connect_at: monthAgo.toISOString(), - sdk: 'powersync-js/1.23.0', + sdk: 'powersync-js/1.23.6', user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', disconnect_at: monthAgo.toISOString(), id: 'month' }; - beforeAll(async () => { - const result = await factory.db.sql` + const user_expired = { + user_id: 'user_expired', + client_id: 'client_expired', + connect_at: monthAgo.toISOString(), + sdk: 'powersync-js/1.23.7', + user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', + jwt_exp: monthAgo.toISOString(), + id: 'expired' + }; + + async function loadData() { + await factory.db.sql` INSERT INTO sdk_report_events ( user_id, @@ -152,8 +172,30 @@ describe('SDK reporting storage', async () => { NULL, ${{ type: 'varchar', value: user_month.id }}, ${{ type: 1184, value: user_month.disconnect_at }} + ), + ( + ${{ type: 'varchar', value: user_expired.user_id }}, + ${{ type: 'varchar', value: user_expired.client_id }}, + ${{ type: 1184, value: user_expired.connect_at }}, + ${{ type: 'varchar', value: user_expired.sdk }}, + ${{ type: 'varchar', value: user_expired.user_agent }}, + ${{ type: 1184, value: user_expired.jwt_exp }}, + ${{ type: 'varchar', value: user_expired.id }}, + NULL ) `.execute(); + } + + function deleteData() { + return factory.db.sql`TRUNCATE TABLE sdk_report_events`.execute(); + } + + beforeAll(async () => { + await loadData(); + }); + + afterAll(async () => { + await deleteData(); }); it('Should show connected users with start range', async () => { const current = await factory.listCurrentConnections({ @@ -205,4 +247,92 @@ describe('SDK reporting storage', async () => { }); expect(sdk).toMatchSnapshot(); }); + + it('Should update a sdk event if its within a day', async () => { + const newConnectAt = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() + 20 + ); + const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); + await factory.reportSdkConnect({ + sdk: user_one.sdk, + connect_at: newConnectAt, + jwt_exp: jwtExp, + client_id: user_one.client_id, + user_id: user_one.user_id, + user_agent: user_one.user_agent + }); + + const sdk = await factory.db + .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_one.user_id }}`.rows(); + expect(sdk).toHaveLength(1); + expect(new Date(sdk[0].connect_at).toISOString()).toEqual(newConnectAt.toISOString()); + expect(new Date(sdk[0].jwt_exp!).toISOString()).toEqual(jwtExp.toISOString()); + expect(sdk[0].disconnect_at).toBeNull(); + const cleaned = removeVolatileFields(sdk); + expect(cleaned).toMatchSnapshot(); + }); + + it('Should update a connected sdk event and make it disconnected', async () => { + const disconnectAt = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() + 20 + ); + const jwtExp = new Date(disconnectAt.getFullYear(), disconnectAt.getMonth(), disconnectAt.getDate() + 1); + + await factory.reportSdkDisconnect({ + disconnect_at: disconnectAt, + jwt_exp: jwtExp, + client_id: user_one.client_id, + user_id: user_one.user_id, + user_agent: user_one.user_agent + }); + + const sdk = await factory.db + .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_one.user_id }}`.rows(); + expect(sdk).toHaveLength(1); + expect(new Date(sdk[0].disconnect_at!).toISOString()).toEqual(disconnectAt.toISOString()); + const cleaned = removeVolatileFields(sdk); + expect(cleaned).toMatchSnapshot(); + }); + + it('Should create a sdk event if its after a day', async () => { + const newConnectAt = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, now.getHours()); + const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); + + await factory.reportSdkConnect({ + sdk: user_week.sdk, + connect_at: newConnectAt, + jwt_exp: jwtExp, + client_id: user_week.client_id, + user_id: user_week.user_id, + user_agent: user_week.user_agent + }); + + const sdk = await factory.db + .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_week.user_id }}`.rows(); + expect(sdk).toHaveLength(2); + const cleaned = removeVolatileFields(sdk); + expect(cleaned).toMatchSnapshot(); + }); + + it('Should delete rows older than specified range', async () => { + await deleteData(); + await loadData(); + await factory.deleteOldSdkData({ + interval: 1, + timeframe: 'week' + }); + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'month' + }); + expect(sdk).toMatchSnapshot(); + }); }); diff --git a/modules/module-postgres-storage/test/src/util.ts b/modules/module-postgres-storage/test/src/util.ts index 19b81106e..825be2ee1 100644 --- a/modules/module-postgres-storage/test/src/util.ts +++ b/modules/module-postgres-storage/test/src/util.ts @@ -1,8 +1,6 @@ import path from 'path'; import { fileURLToPath } from 'url'; -import { normalizePostgresStorageConfig } from '../../src//types/types.js'; -import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js'; -import { postgresTestSetup } from '../../src/storage/PostgresTestStorageFactoryGenerator.js'; +import { normalizePostgresStorageConfig, PostgresMigrationAgent, postgresTestSetup } from '../../src/index.js'; import { env } from './env.js'; const __filename = fileURLToPath(import.meta.url); diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index f5700f5ce..3a03604ce 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -43,6 +43,7 @@ export type SdkDisconnectEventData = { } & SdkUserData; export type SdkConnectDocument = { + id: string; sdk: string; user_agent: string; client_id: string; From 794a0990d5af9bdd9ab24ab8becb1119c0a33307 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 4 Aug 2025 08:55:16 +0200 Subject: [PATCH 143/169] mongo tests report --- .../MongoTestReportStorageFactoryGenerator.ts | 22 ++ .../test/src/sdk-report-storage.test.ts | 253 ++++++++++++++++++ .../module-mongodb-storage/test/src/util.ts | 6 + .../storage/PostgresReportStorageFactory.ts | 2 +- packages/types/src/events.ts | 2 +- 5 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts create mode 100644 modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts new file mode 100644 index 000000000..d3cf0d71b --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts @@ -0,0 +1,22 @@ +import { TestStorageOptions } from '@powersync/service-core'; +import { connectMongoForTests } from './util.js'; +import { MongoReportStorage } from '../MongoReportStorage.js'; + +export type MongoTestStorageOptions = { + url: string; + isCI: boolean; +}; + +export const MongoTestReportStorageFactoryGenerator = (factoryOptions: MongoTestStorageOptions) => { + return async (options?: TestStorageOptions) => { + const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); + + await db.createSdkReportingCollection(); + + if (!options?.doNotClear) { + await db.clear(); + } + + return new MongoReportStorage(db); + }; +}; diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts new file mode 100644 index 000000000..750793f95 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts @@ -0,0 +1,253 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js'; +import { event_types } from '@powersync/service-types'; + +function removeVolatileFields(sdks: event_types.SdkConnectDocument[]): Partial[] { + return sdks.map((sdk) => { + const { id, disconnect_at, connect_at, jwt_exp, ...rest } = sdk; + return { + ...rest + }; + }); +} + +describe('SDK reporting storage', async () => { + const factory = await INITIALIZED_MONGO_REPORT_STORAGE_FACTORY(); + const now = new Date(); + const nowAdd5minutes = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() + 5 + ); + const nowLess5minutes = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() - 5 + ); + const yesterday = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1); + const weekAgo = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7); + const monthAgo = new Date(now.getFullYear(), now.getMonth() - 1, now.getDate()); + const user_one = { + user_id: 'user_one', + client_id: 'client_one', + connect_at: now, + sdk: 'powersync-dart/1.6.4', + user_agent: 'powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android', + jwt_exp: nowAdd5minutes + }; + const user_two = { + user_id: 'user_two', + client_id: 'client_two', + connect_at: nowLess5minutes, + sdk: 'powersync-js/1.21.1', + user_agent: 'powersync-js/1.21.0 powersync-web Chromium/138 linux', + jwt_exp: nowAdd5minutes + }; + const user_three = { + user_id: 'user_three', + client_id: 'client_three', + connect_at: yesterday, + sdk: 'powersync-js/1.21.2', + user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', + disconnect_at: yesterday + }; + + const user_four = { + user_id: 'user_four', + client_id: 'client_four', + connect_at: now, + sdk: 'powersync-js/1.21.4', + user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', + jwt_exp: nowLess5minutes + }; + + const user_week = { + user_id: 'user_week', + client_id: 'client_week', + connect_at: weekAgo, + sdk: 'powersync-js/1.24.5', + user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', + disconnect_at: weekAgo + }; + + const user_month = { + user_id: 'user_month', + client_id: 'client_month', + connect_at: monthAgo, + sdk: 'powersync-js/1.23.6', + user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', + disconnect_at: monthAgo + }; + + const user_expired = { + user_id: 'user_expired', + client_id: 'client_expired', + connect_at: monthAgo, + sdk: 'powersync-js/1.23.7', + user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', + jwt_exp: monthAgo + }; + + async function loadData() { + await factory.db.sdk_report_events.insertMany([ + user_one, + user_two, + user_three, + user_four, + user_week, + user_month, + user_expired + ]); + } + + function deleteData() { + return factory.db.sdk_report_events.deleteMany(); + } + + beforeAll(async () => { + await loadData(); + }); + + afterAll(async () => { + await deleteData(); + }); + it('Should show connected users with start range', async () => { + const current = await factory.listCurrentConnections({ + range: { + start_date: new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() - 1 + ).toISOString() + } + }); + expect(current).toMatchSnapshot(); + }); + it('Should show connected users with start range and end range', async () => { + const current = await factory.listCurrentConnections({ + range: { + end_date: nowLess5minutes.toISOString(), + start_date: new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() - 6 + ).toISOString() + } + }); + expect(current).toMatchSnapshot(); + }); + it('Should show SDK scrape data for user over the past month', async () => { + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'month' + }); + expect(sdk).toMatchSnapshot(); + }); + it('Should show SDK scrape data for user over the past week', async () => { + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'week' + }); + expect(sdk).toMatchSnapshot(); + }); + it('Should show SDK scrape data for user over the past day', async () => { + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'day' + }); + expect(sdk).toMatchSnapshot(); + }); + + it('Should update a sdk event if its within a day', async () => { + const newConnectAt = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() + 20 + ); + const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); + await factory.reportSdkConnect({ + sdk: user_one.sdk, + connect_at: newConnectAt, + jwt_exp: jwtExp, + client_id: user_one.client_id, + user_id: user_one.user_id, + user_agent: user_one.user_agent + }); + + const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); + expect(sdk).toHaveLength(1); + expect(new Date(sdk[0].connect_at)).toEqual(newConnectAt); + expect(new Date(sdk[0].jwt_exp!)).toEqual(jwtExp); + expect(sdk[0].disconnect_at).toBeNull(); + const cleaned = removeVolatileFields(sdk); + expect(cleaned).toMatchSnapshot(); + }); + + it('Should update a connected sdk event and make it disconnected', async () => { + const disconnectAt = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() + 20 + ); + const jwtExp = new Date(disconnectAt.getFullYear(), disconnectAt.getMonth(), disconnectAt.getDate() + 1); + + await factory.reportSdkDisconnect({ + disconnect_at: disconnectAt, + jwt_exp: jwtExp, + client_id: user_one.client_id, + user_id: user_one.user_id, + user_agent: user_one.user_agent + }); + + const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); + expect(sdk).toHaveLength(1); + expect(new Date(sdk[0].disconnect_at!)).toEqual(disconnectAt); + const cleaned = removeVolatileFields(sdk); + expect(cleaned).toMatchSnapshot(); + }); + + it('Should create a sdk event if its after a day', async () => { + const newConnectAt = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, now.getHours()); + const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); + + await factory.reportSdkConnect({ + sdk: user_week.sdk, + connect_at: newConnectAt, + jwt_exp: jwtExp, + client_id: user_week.client_id, + user_id: user_week.user_id, + user_agent: user_week.user_agent + }); + + const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); + expect(sdk).toHaveLength(2); + const cleaned = removeVolatileFields(sdk); + expect(cleaned).toMatchSnapshot(); + }); + + it('Should delete rows older than specified range', async () => { + await deleteData(); + await loadData(); + await factory.deleteOldSdkData({ + interval: 1, + timeframe: 'week' + }); + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'month' + }); + expect(sdk).toMatchSnapshot(); + }); +}); diff --git a/modules/module-mongodb-storage/test/src/util.ts b/modules/module-mongodb-storage/test/src/util.ts index db3b2dd18..ae5672c57 100644 --- a/modules/module-mongodb-storage/test/src/util.ts +++ b/modules/module-mongodb-storage/test/src/util.ts @@ -1,8 +1,14 @@ import { env } from './env.js'; import { MongoTestStorageFactoryGenerator } from '@module/storage/implementation/MongoTestStorageFactoryGenerator.js'; +import { MongoTestReportStorageFactoryGenerator } from '@module/storage/implementation/MongoTestReportStorageFactoryGenerator.js'; export const INITIALIZED_MONGO_STORAGE_FACTORY = MongoTestStorageFactoryGenerator({ url: env.MONGO_TEST_URL, isCI: env.CI }); + +export const INITIALIZED_MONGO_REPORT_STORAGE_FACTORY = MongoTestReportStorageFactoryGenerator({ + url: env.MONGO_TEST_URL, + isCI: env.CI +}); diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index ff589bbc1..ca6399f43 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -367,7 +367,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } async [Symbol.asyncDispose]() { - // await this.db[Symbol.asyncDispose](); + await this.db[Symbol.asyncDispose](); } async prepareStatements(connection: pg_wire.PgConnection) { diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 3a03604ce..eb6918949 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -43,7 +43,7 @@ export type SdkDisconnectEventData = { } & SdkUserData; export type SdkConnectDocument = { - id: string; + id?: string; sdk: string; user_agent: string; client_id: string; From 9b7fde8bfa22ab6ca6facde364c87d0fc117f384 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 4 Aug 2025 09:21:58 +0200 Subject: [PATCH 144/169] seperated the migrations for sdk, problems testing --- .../test/src/sdk-report-storage.test.ts | 253 ------------------ .../test/src/sdk-report-storage.ts | 253 ++++++++++++++++++ .../migrations/scripts/1684951997326-init.ts | 20 -- .../scripts/1753871047719-sdk_report.ts | 45 ++++ 4 files changed, 298 insertions(+), 273 deletions(-) delete mode 100644 modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts create mode 100644 modules/module-mongodb-storage/test/src/sdk-report-storage.ts create mode 100644 modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts deleted file mode 100644 index 750793f95..000000000 --- a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js'; -import { event_types } from '@powersync/service-types'; - -function removeVolatileFields(sdks: event_types.SdkConnectDocument[]): Partial[] { - return sdks.map((sdk) => { - const { id, disconnect_at, connect_at, jwt_exp, ...rest } = sdk; - return { - ...rest - }; - }); -} - -describe('SDK reporting storage', async () => { - const factory = await INITIALIZED_MONGO_REPORT_STORAGE_FACTORY(); - const now = new Date(); - const nowAdd5minutes = new Date( - now.getFullYear(), - now.getMonth(), - now.getDate(), - now.getHours(), - now.getMinutes() + 5 - ); - const nowLess5minutes = new Date( - now.getFullYear(), - now.getMonth(), - now.getDate(), - now.getHours(), - now.getMinutes() - 5 - ); - const yesterday = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1); - const weekAgo = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7); - const monthAgo = new Date(now.getFullYear(), now.getMonth() - 1, now.getDate()); - const user_one = { - user_id: 'user_one', - client_id: 'client_one', - connect_at: now, - sdk: 'powersync-dart/1.6.4', - user_agent: 'powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android', - jwt_exp: nowAdd5minutes - }; - const user_two = { - user_id: 'user_two', - client_id: 'client_two', - connect_at: nowLess5minutes, - sdk: 'powersync-js/1.21.1', - user_agent: 'powersync-js/1.21.0 powersync-web Chromium/138 linux', - jwt_exp: nowAdd5minutes - }; - const user_three = { - user_id: 'user_three', - client_id: 'client_three', - connect_at: yesterday, - sdk: 'powersync-js/1.21.2', - user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', - disconnect_at: yesterday - }; - - const user_four = { - user_id: 'user_four', - client_id: 'client_four', - connect_at: now, - sdk: 'powersync-js/1.21.4', - user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', - jwt_exp: nowLess5minutes - }; - - const user_week = { - user_id: 'user_week', - client_id: 'client_week', - connect_at: weekAgo, - sdk: 'powersync-js/1.24.5', - user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', - disconnect_at: weekAgo - }; - - const user_month = { - user_id: 'user_month', - client_id: 'client_month', - connect_at: monthAgo, - sdk: 'powersync-js/1.23.6', - user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', - disconnect_at: monthAgo - }; - - const user_expired = { - user_id: 'user_expired', - client_id: 'client_expired', - connect_at: monthAgo, - sdk: 'powersync-js/1.23.7', - user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', - jwt_exp: monthAgo - }; - - async function loadData() { - await factory.db.sdk_report_events.insertMany([ - user_one, - user_two, - user_three, - user_four, - user_week, - user_month, - user_expired - ]); - } - - function deleteData() { - return factory.db.sdk_report_events.deleteMany(); - } - - beforeAll(async () => { - await loadData(); - }); - - afterAll(async () => { - await deleteData(); - }); - it('Should show connected users with start range', async () => { - const current = await factory.listCurrentConnections({ - range: { - start_date: new Date( - now.getFullYear(), - now.getMonth(), - now.getDate(), - now.getHours(), - now.getMinutes() - 1 - ).toISOString() - } - }); - expect(current).toMatchSnapshot(); - }); - it('Should show connected users with start range and end range', async () => { - const current = await factory.listCurrentConnections({ - range: { - end_date: nowLess5minutes.toISOString(), - start_date: new Date( - now.getFullYear(), - now.getMonth(), - now.getDate(), - now.getHours(), - now.getMinutes() - 6 - ).toISOString() - } - }); - expect(current).toMatchSnapshot(); - }); - it('Should show SDK scrape data for user over the past month', async () => { - const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'month' - }); - expect(sdk).toMatchSnapshot(); - }); - it('Should show SDK scrape data for user over the past week', async () => { - const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'week' - }); - expect(sdk).toMatchSnapshot(); - }); - it('Should show SDK scrape data for user over the past day', async () => { - const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'day' - }); - expect(sdk).toMatchSnapshot(); - }); - - it('Should update a sdk event if its within a day', async () => { - const newConnectAt = new Date( - now.getFullYear(), - now.getMonth(), - now.getDate(), - now.getHours(), - now.getMinutes() + 20 - ); - const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); - await factory.reportSdkConnect({ - sdk: user_one.sdk, - connect_at: newConnectAt, - jwt_exp: jwtExp, - client_id: user_one.client_id, - user_id: user_one.user_id, - user_agent: user_one.user_agent - }); - - const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); - expect(sdk).toHaveLength(1); - expect(new Date(sdk[0].connect_at)).toEqual(newConnectAt); - expect(new Date(sdk[0].jwt_exp!)).toEqual(jwtExp); - expect(sdk[0].disconnect_at).toBeNull(); - const cleaned = removeVolatileFields(sdk); - expect(cleaned).toMatchSnapshot(); - }); - - it('Should update a connected sdk event and make it disconnected', async () => { - const disconnectAt = new Date( - now.getFullYear(), - now.getMonth(), - now.getDate(), - now.getHours(), - now.getMinutes() + 20 - ); - const jwtExp = new Date(disconnectAt.getFullYear(), disconnectAt.getMonth(), disconnectAt.getDate() + 1); - - await factory.reportSdkDisconnect({ - disconnect_at: disconnectAt, - jwt_exp: jwtExp, - client_id: user_one.client_id, - user_id: user_one.user_id, - user_agent: user_one.user_agent - }); - - const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); - expect(sdk).toHaveLength(1); - expect(new Date(sdk[0].disconnect_at!)).toEqual(disconnectAt); - const cleaned = removeVolatileFields(sdk); - expect(cleaned).toMatchSnapshot(); - }); - - it('Should create a sdk event if its after a day', async () => { - const newConnectAt = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, now.getHours()); - const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); - - await factory.reportSdkConnect({ - sdk: user_week.sdk, - connect_at: newConnectAt, - jwt_exp: jwtExp, - client_id: user_week.client_id, - user_id: user_week.user_id, - user_agent: user_week.user_agent - }); - - const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); - expect(sdk).toHaveLength(2); - const cleaned = removeVolatileFields(sdk); - expect(cleaned).toMatchSnapshot(); - }); - - it('Should delete rows older than specified range', async () => { - await deleteData(); - await loadData(); - await factory.deleteOldSdkData({ - interval: 1, - timeframe: 'week' - }); - const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'month' - }); - expect(sdk).toMatchSnapshot(); - }); -}); diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.ts b/modules/module-mongodb-storage/test/src/sdk-report-storage.ts new file mode 100644 index 000000000..329f2321f --- /dev/null +++ b/modules/module-mongodb-storage/test/src/sdk-report-storage.ts @@ -0,0 +1,253 @@ +// import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +// import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js'; +// import { event_types } from '@powersync/service-types'; +// +// function removeVolatileFields(sdks: event_types.SdkConnectDocument[]): Partial[] { +// return sdks.map((sdk) => { +// const { id, disconnect_at, connect_at, jwt_exp, ...rest } = sdk; +// return { +// ...rest +// }; +// }); +// } +// +// describe('SDK reporting storage', async () => { +// const factory = await INITIALIZED_MONGO_REPORT_STORAGE_FACTORY(); +// const now = new Date(); +// const nowAdd5minutes = new Date( +// now.getFullYear(), +// now.getMonth(), +// now.getDate(), +// now.getHours(), +// now.getMinutes() + 5 +// ); +// const nowLess5minutes = new Date( +// now.getFullYear(), +// now.getMonth(), +// now.getDate(), +// now.getHours(), +// now.getMinutes() - 5 +// ); +// const yesterday = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1); +// const weekAgo = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7); +// const monthAgo = new Date(now.getFullYear(), now.getMonth() - 1, now.getDate()); +// const user_one = { +// user_id: 'user_one', +// client_id: 'client_one', +// connect_at: now, +// sdk: 'powersync-dart/1.6.4', +// user_agent: 'powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android', +// jwt_exp: nowAdd5minutes +// }; +// const user_two = { +// user_id: 'user_two', +// client_id: 'client_two', +// connect_at: nowLess5minutes, +// sdk: 'powersync-js/1.21.1', +// user_agent: 'powersync-js/1.21.0 powersync-web Chromium/138 linux', +// jwt_exp: nowAdd5minutes +// }; +// const user_three = { +// user_id: 'user_three', +// client_id: 'client_three', +// connect_at: yesterday, +// sdk: 'powersync-js/1.21.2', +// user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', +// disconnect_at: yesterday +// }; +// +// const user_four = { +// user_id: 'user_four', +// client_id: 'client_four', +// connect_at: now, +// sdk: 'powersync-js/1.21.4', +// user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', +// jwt_exp: nowLess5minutes +// }; +// +// const user_week = { +// user_id: 'user_week', +// client_id: 'client_week', +// connect_at: weekAgo, +// sdk: 'powersync-js/1.24.5', +// user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', +// disconnect_at: weekAgo +// }; +// +// const user_month = { +// user_id: 'user_month', +// client_id: 'client_month', +// connect_at: monthAgo, +// sdk: 'powersync-js/1.23.6', +// user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', +// disconnect_at: monthAgo +// }; +// +// const user_expired = { +// user_id: 'user_expired', +// client_id: 'client_expired', +// connect_at: monthAgo, +// sdk: 'powersync-js/1.23.7', +// user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', +// jwt_exp: monthAgo +// }; +// +// async function loadData() { +// await factory.db.sdk_report_events.insertMany([ +// user_one, +// user_two, +// user_three, +// user_four, +// user_week, +// user_month, +// user_expired +// ]); +// } +// +// function deleteData() { +// return factory.db.sdk_report_events.deleteMany(); +// } +// +// beforeAll(async () => { +// await loadData(); +// }); +// +// afterAll(async () => { +// await deleteData(); +// }); +// it('Should show connected users with start range', async () => { +// const current = await factory.listCurrentConnections({ +// range: { +// start_date: new Date( +// now.getFullYear(), +// now.getMonth(), +// now.getDate(), +// now.getHours(), +// now.getMinutes() - 1 +// ).toISOString() +// } +// }); +// expect(current).toMatchSnapshot(); +// }); +// it('Should show connected users with start range and end range', async () => { +// const current = await factory.listCurrentConnections({ +// range: { +// end_date: nowLess5minutes.toISOString(), +// start_date: new Date( +// now.getFullYear(), +// now.getMonth(), +// now.getDate(), +// now.getHours(), +// now.getMinutes() - 6 +// ).toISOString() +// } +// }); +// expect(current).toMatchSnapshot(); +// }); +// it('Should show SDK scrape data for user over the past month', async () => { +// const sdk = await factory.scrapeSdkData({ +// interval: 1, +// timeframe: 'month' +// }); +// expect(sdk).toMatchSnapshot(); +// }); +// it('Should show SDK scrape data for user over the past week', async () => { +// const sdk = await factory.scrapeSdkData({ +// interval: 1, +// timeframe: 'week' +// }); +// expect(sdk).toMatchSnapshot(); +// }); +// it('Should show SDK scrape data for user over the past day', async () => { +// const sdk = await factory.scrapeSdkData({ +// interval: 1, +// timeframe: 'day' +// }); +// expect(sdk).toMatchSnapshot(); +// }); +// +// it('Should update a sdk event if its within a day', async () => { +// const newConnectAt = new Date( +// now.getFullYear(), +// now.getMonth(), +// now.getDate(), +// now.getHours(), +// now.getMinutes() + 20 +// ); +// const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); +// await factory.reportSdkConnect({ +// sdk: user_one.sdk, +// connect_at: newConnectAt, +// jwt_exp: jwtExp, +// client_id: user_one.client_id, +// user_id: user_one.user_id, +// user_agent: user_one.user_agent +// }); +// +// const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); +// expect(sdk).toHaveLength(1); +// expect(new Date(sdk[0].connect_at)).toEqual(newConnectAt); +// expect(new Date(sdk[0].jwt_exp!)).toEqual(jwtExp); +// expect(sdk[0].disconnect_at).toBeNull(); +// const cleaned = removeVolatileFields(sdk); +// expect(cleaned).toMatchSnapshot(); +// }); +// +// it('Should update a connected sdk event and make it disconnected', async () => { +// const disconnectAt = new Date( +// now.getFullYear(), +// now.getMonth(), +// now.getDate(), +// now.getHours(), +// now.getMinutes() + 20 +// ); +// const jwtExp = new Date(disconnectAt.getFullYear(), disconnectAt.getMonth(), disconnectAt.getDate() + 1); +// +// await factory.reportSdkDisconnect({ +// disconnect_at: disconnectAt, +// jwt_exp: jwtExp, +// client_id: user_one.client_id, +// user_id: user_one.user_id, +// user_agent: user_one.user_agent +// }); +// +// const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); +// expect(sdk).toHaveLength(1); +// expect(new Date(sdk[0].disconnect_at!)).toEqual(disconnectAt); +// const cleaned = removeVolatileFields(sdk); +// expect(cleaned).toMatchSnapshot(); +// }); +// +// it('Should create a sdk event if its after a day', async () => { +// const newConnectAt = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, now.getHours()); +// const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); +// +// await factory.reportSdkConnect({ +// sdk: user_week.sdk, +// connect_at: newConnectAt, +// jwt_exp: jwtExp, +// client_id: user_week.client_id, +// user_id: user_week.user_id, +// user_agent: user_week.user_agent +// }); +// +// const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); +// expect(sdk).toHaveLength(2); +// const cleaned = removeVolatileFields(sdk); +// expect(cleaned).toMatchSnapshot(); +// }); +// +// it('Should delete rows older than specified range', async () => { +// await deleteData(); +// await loadData(); +// await factory.deleteOldSdkData({ +// interval: 1, +// timeframe: 'week' +// }); +// const sdk = await factory.scrapeSdkData({ +// interval: 1, +// timeframe: 'month' +// }); +// expect(sdk).toMatchSnapshot(); +// }); +// }); diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index d54669d11..61e8699ea 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -128,26 +128,6 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { CONSTRAINT unique_user_sync PRIMARY KEY (user_id, sync_rules_id) ); `.execute(); - await db.sql` - CREATE TABLE sdk_report_events ( - id TEXT PRIMARY KEY, - user_agent TEXT NOT NULL, - client_id TEXT NOT NULL, - user_id TEXT NOT NULL, - sdk TEXT NOT NULL, - jwt_exp TIMESTAMP WITH TIME ZONE, - connect_at TIMESTAMP WITH TIME ZONE NOT NULL, - disconnect_at TIMESTAMP WITH TIME ZONE - ) - `.execute(); - - await db.sql` CREATE INDEX sdk_list_index ON sdk_report_events (connect_at, jwt_exp, disconnect_at) `.execute(); - - await db.sql`CREATE INDEX sdk_user_id_index ON sdk_report_events (user_id)`.execute(); - - await db.sql`CREATE INDEX sdk_client_id_index ON sdk_report_events (client_id)`.execute(); - - await db.sql`CREATE INDEX sdk_index ON sdk_report_events (sdk)`.execute(); }); }; diff --git a/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts b/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts new file mode 100644 index 000000000..c19f2f921 --- /dev/null +++ b/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts @@ -0,0 +1,45 @@ +import { migrations } from '@powersync/service-core'; + +import { openMigrationDB } from '../migration-utils.js'; + +export const up: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + await using client = openMigrationDB(configuration.storage); + + await client.transaction(async (db) => { + await db.sql` + CREATE TABLE IF NOT EXISTS sdk_report_events ( + id TEXT PRIMARY KEY, + user_agent TEXT NOT NULL, + client_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sdk TEXT NOT NULL, + jwt_exp TIMESTAMP WITH TIME ZONE, + connect_at TIMESTAMP WITH TIME ZONE NOT NULL, + disconnect_at TIMESTAMP WITH TIME ZONE + ) + `.execute(); + + await db.sql` + CREATE INDEX IF NOT EXISTS sdk_list_index ON sdk_report_events (connect_at, jwt_exp, disconnect_at) + `.execute(); + + await db.sql`CREATE INDEX IF NOT EXISTS sdk_user_id_index ON sdk_report_events (user_id)`.execute(); + + await db.sql`CREATE INDEX IF NOT EXISTS sdk_client_id_index ON sdk_report_events (client_id)`.execute(); + + await db.sql`CREATE INDEX IF NOT EXISTS sdk_index ON sdk_report_events (sdk)`.execute(); + }); +}; + +export const down: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + await using client = openMigrationDB(configuration.storage); + client.lockConnection(async (db) => { + await db.sql`DROP TABLE IF EXISTS sdk_report_events`.execute(); + }); +}; From 1af5e3aa636a5aeb5cb038b291d186290acf778e Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 4 Aug 2025 09:30:59 +0200 Subject: [PATCH 145/169] okay then --- .../migrations/scripts/1684951997326-init.ts | 20 +++++++++ .../scripts/1753871047719-sdk_report.ts | 45 ------------------- 2 files changed, 20 insertions(+), 45 deletions(-) delete mode 100644 modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index 61e8699ea..d54669d11 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -128,6 +128,26 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { CONSTRAINT unique_user_sync PRIMARY KEY (user_id, sync_rules_id) ); `.execute(); + await db.sql` + CREATE TABLE sdk_report_events ( + id TEXT PRIMARY KEY, + user_agent TEXT NOT NULL, + client_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sdk TEXT NOT NULL, + jwt_exp TIMESTAMP WITH TIME ZONE, + connect_at TIMESTAMP WITH TIME ZONE NOT NULL, + disconnect_at TIMESTAMP WITH TIME ZONE + ) + `.execute(); + + await db.sql` CREATE INDEX sdk_list_index ON sdk_report_events (connect_at, jwt_exp, disconnect_at) `.execute(); + + await db.sql`CREATE INDEX sdk_user_id_index ON sdk_report_events (user_id)`.execute(); + + await db.sql`CREATE INDEX sdk_client_id_index ON sdk_report_events (client_id)`.execute(); + + await db.sql`CREATE INDEX sdk_index ON sdk_report_events (sdk)`.execute(); }); }; diff --git a/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts b/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts deleted file mode 100644 index c19f2f921..000000000 --- a/modules/module-postgres-storage/src/migrations/scripts/1753871047719-sdk_report.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { migrations } from '@powersync/service-core'; - -import { openMigrationDB } from '../migration-utils.js'; - -export const up: migrations.PowerSyncMigrationFunction = async (context) => { - const { - service_context: { configuration } - } = context; - await using client = openMigrationDB(configuration.storage); - - await client.transaction(async (db) => { - await db.sql` - CREATE TABLE IF NOT EXISTS sdk_report_events ( - id TEXT PRIMARY KEY, - user_agent TEXT NOT NULL, - client_id TEXT NOT NULL, - user_id TEXT NOT NULL, - sdk TEXT NOT NULL, - jwt_exp TIMESTAMP WITH TIME ZONE, - connect_at TIMESTAMP WITH TIME ZONE NOT NULL, - disconnect_at TIMESTAMP WITH TIME ZONE - ) - `.execute(); - - await db.sql` - CREATE INDEX IF NOT EXISTS sdk_list_index ON sdk_report_events (connect_at, jwt_exp, disconnect_at) - `.execute(); - - await db.sql`CREATE INDEX IF NOT EXISTS sdk_user_id_index ON sdk_report_events (user_id)`.execute(); - - await db.sql`CREATE INDEX IF NOT EXISTS sdk_client_id_index ON sdk_report_events (client_id)`.execute(); - - await db.sql`CREATE INDEX IF NOT EXISTS sdk_index ON sdk_report_events (sdk)`.execute(); - }); -}; - -export const down: migrations.PowerSyncMigrationFunction = async (context) => { - const { - service_context: { configuration } - } = context; - await using client = openMigrationDB(configuration.storage); - client.lockConnection(async (db) => { - await db.sql`DROP TABLE IF EXISTS sdk_report_events`.execute(); - }); -}; From 30d0efd4d917f647ce94f826470a6466b17c93b2 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 4 Aug 2025 10:13:23 +0200 Subject: [PATCH 146/169] mongo db tests report storage --- .../src/storage/MongoReportStorage.ts | 5 + .../sdk-report-storage.test.ts.snap | 188 +++++++++++++ .../test/src/sdk-report-storage.test.ts | 255 ++++++++++++++++++ .../test/src/sdk-report-storage.ts | 253 ----------------- 4 files changed, 448 insertions(+), 253 deletions(-) create mode 100644 modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap create mode 100644 modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts delete mode 100644 modules/module-mongodb-storage/test/src/sdk-report-storage.ts diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 3cc038eae..cb59e0b33 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -56,6 +56,11 @@ export class MongoReportStorage implements storage.ReportStorageFactory { users: { $size: '$user_ids' }, clients: { $size: '$client_ids' } } + }, + { + $sort: { + sdk: 1 + } } ] } diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap new file mode 100644 index 000000000..7ddd94b2d --- /dev/null +++ b/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap @@ -0,0 +1,188 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`SDK reporting storage > Should create a sdk event if its after a day 1`] = ` +[ + { + "client_id": "client_week", + "sdk": "powersync-js/1.24.5", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_week", + }, + { + "client_id": "client_week", + "sdk": "powersync-js/1.24.5", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_week", + }, +] +`; + +exports[`SDK reporting storage > Should delete rows older than specified range 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + ], + "users": 4, +} +`; + +exports[`SDK reporting storage > Should show SDK scrape data for user over the past day 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + ], + "users": 3, +} +`; + +exports[`SDK reporting storage > Should show SDK scrape data for user over the past month 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.23.6", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.23.7", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, + }, + ], + "users": 7, +} +`; + +exports[`SDK reporting storage > Should show SDK scrape data for user over the past week 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + ], + "users": 4, +} +`; + +exports[`SDK reporting storage > Should show connected users with start range 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + ], + "users": 1, +} +`; + +exports[`SDK reporting storage > Should show connected users with start range and end range 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + ], + "users": 1, +} +`; + +exports[`SDK reporting storage > Should update a connected sdk event and make it disconnected 1`] = ` +[ + { + "client_id": "client_one", + "sdk": "powersync-dart/1.6.4", + "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android", + "user_id": "user_one", + }, +] +`; + +exports[`SDK reporting storage > Should update a sdk event if its within a day 1`] = ` +[ + { + "client_id": "client_one", + "sdk": "powersync-dart/1.6.4", + "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android", + "user_id": "user_one", + }, +] +`; diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts new file mode 100644 index 000000000..60c8cc502 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts @@ -0,0 +1,255 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js'; +import { event_types } from '@powersync/service-types'; + +function removeVolatileFields( + sdks: event_types.SdkConnectDocument[] +): Partial[] { + return sdks.map((sdk: Partial) => { + const { _id, disconnect_at, connect_at, jwt_exp, ...rest } = sdk; + return { + ...rest + }; + }); +} + +describe('SDK reporting storage', async () => { + const factory = await INITIALIZED_MONGO_REPORT_STORAGE_FACTORY(); + const now = new Date(); + const nowAdd5minutes = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() + 5 + ); + const nowLess5minutes = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() - 5 + ); + const yesterday = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1); + const weekAgo = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7); + const monthAgo = new Date(now.getFullYear(), now.getMonth() - 1, now.getDate()); + const user_one = { + user_id: 'user_one', + client_id: 'client_one', + connect_at: now, + sdk: 'powersync-dart/1.6.4', + user_agent: 'powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android', + jwt_exp: nowAdd5minutes + }; + const user_two = { + user_id: 'user_two', + client_id: 'client_two', + connect_at: nowLess5minutes, + sdk: 'powersync-js/1.21.1', + user_agent: 'powersync-js/1.21.0 powersync-web Chromium/138 linux', + jwt_exp: nowAdd5minutes + }; + const user_three = { + user_id: 'user_three', + client_id: 'client_three', + connect_at: yesterday, + sdk: 'powersync-js/1.21.2', + user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', + disconnect_at: yesterday + }; + + const user_four = { + user_id: 'user_four', + client_id: 'client_four', + connect_at: now, + sdk: 'powersync-js/1.21.4', + user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', + jwt_exp: nowLess5minutes + }; + + const user_week = { + user_id: 'user_week', + client_id: 'client_week', + connect_at: weekAgo, + sdk: 'powersync-js/1.24.5', + user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', + disconnect_at: weekAgo + }; + + const user_month = { + user_id: 'user_month', + client_id: 'client_month', + connect_at: monthAgo, + sdk: 'powersync-js/1.23.6', + user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', + disconnect_at: monthAgo + }; + + const user_expired = { + user_id: 'user_expired', + client_id: 'client_expired', + connect_at: monthAgo, + sdk: 'powersync-js/1.23.7', + user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', + jwt_exp: monthAgo + }; + + async function loadData() { + await factory.db.sdk_report_events.insertMany([ + user_one, + user_two, + user_three, + user_four, + user_week, + user_month, + user_expired + ]); + } + + function deleteData() { + return factory.db.sdk_report_events.deleteMany(); + } + + beforeAll(async () => { + await loadData(); + }); + + afterAll(async () => { + await deleteData(); + }); + it('Should show connected users with start range', async () => { + const current = await factory.listCurrentConnections({ + range: { + start_date: new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() - 1 + ).toISOString() + } + }); + expect(current).toMatchSnapshot(); + }); + it('Should show connected users with start range and end range', async () => { + const current = await factory.listCurrentConnections({ + range: { + end_date: nowLess5minutes.toISOString(), + start_date: new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() - 6 + ).toISOString() + } + }); + expect(current).toMatchSnapshot(); + }); + it('Should show SDK scrape data for user over the past month', async () => { + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'month' + }); + expect(sdk).toMatchSnapshot(); + }); + it('Should show SDK scrape data for user over the past week', async () => { + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'week' + }); + expect(sdk).toMatchSnapshot(); + }); + it('Should show SDK scrape data for user over the past day', async () => { + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'day' + }); + expect(sdk).toMatchSnapshot(); + }); + + it('Should update a sdk event if its within a day', async () => { + const newConnectAt = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() + 20 + ); + const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); + await factory.reportSdkConnect({ + sdk: user_one.sdk, + connect_at: newConnectAt, + jwt_exp: jwtExp, + client_id: user_one.client_id, + user_id: user_one.user_id, + user_agent: user_one.user_agent + }); + + const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); + expect(sdk).toHaveLength(1); + expect(new Date(sdk[0].connect_at)).toEqual(newConnectAt); + expect(new Date(sdk[0].jwt_exp!)).toEqual(jwtExp); + expect(sdk[0].disconnect_at).toBeUndefined(); + const cleaned = removeVolatileFields(sdk); + expect(cleaned).toMatchSnapshot(); + }); + + it('Should update a connected sdk event and make it disconnected', async () => { + const disconnectAt = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate(), + now.getHours(), + now.getMinutes() + 20 + ); + const jwtExp = new Date(disconnectAt.getFullYear(), disconnectAt.getMonth(), disconnectAt.getDate() + 1); + + await factory.reportSdkDisconnect({ + disconnect_at: disconnectAt, + jwt_exp: jwtExp, + client_id: user_one.client_id, + user_id: user_one.user_id, + user_agent: user_one.user_agent + }); + + const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); + expect(sdk).toHaveLength(1); + expect(new Date(sdk[0].disconnect_at!)).toEqual(disconnectAt); + const cleaned = removeVolatileFields(sdk); + expect(cleaned).toMatchSnapshot(); + }); + + it('Should create a sdk event if its after a day', async () => { + const newConnectAt = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, now.getHours()); + const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); + + await factory.reportSdkConnect({ + sdk: user_week.sdk, + connect_at: newConnectAt, + jwt_exp: jwtExp, + client_id: user_week.client_id, + user_id: user_week.user_id, + user_agent: user_week.user_agent + }); + + const sdk = await factory.db.sdk_report_events.find({ user_id: user_week.user_id }).toArray(); + expect(sdk).toHaveLength(2); + const cleaned = removeVolatileFields(sdk); + expect(cleaned).toMatchSnapshot(); + }); + + it('Should delete rows older than specified range', async () => { + await deleteData(); + await loadData(); + await factory.deleteOldSdkData({ + interval: 1, + timeframe: 'week' + }); + const sdk = await factory.scrapeSdkData({ + interval: 1, + timeframe: 'month' + }); + expect(sdk).toMatchSnapshot(); + }); +}); diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.ts b/modules/module-mongodb-storage/test/src/sdk-report-storage.ts deleted file mode 100644 index 329f2321f..000000000 --- a/modules/module-mongodb-storage/test/src/sdk-report-storage.ts +++ /dev/null @@ -1,253 +0,0 @@ -// import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -// import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js'; -// import { event_types } from '@powersync/service-types'; -// -// function removeVolatileFields(sdks: event_types.SdkConnectDocument[]): Partial[] { -// return sdks.map((sdk) => { -// const { id, disconnect_at, connect_at, jwt_exp, ...rest } = sdk; -// return { -// ...rest -// }; -// }); -// } -// -// describe('SDK reporting storage', async () => { -// const factory = await INITIALIZED_MONGO_REPORT_STORAGE_FACTORY(); -// const now = new Date(); -// const nowAdd5minutes = new Date( -// now.getFullYear(), -// now.getMonth(), -// now.getDate(), -// now.getHours(), -// now.getMinutes() + 5 -// ); -// const nowLess5minutes = new Date( -// now.getFullYear(), -// now.getMonth(), -// now.getDate(), -// now.getHours(), -// now.getMinutes() - 5 -// ); -// const yesterday = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1); -// const weekAgo = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7); -// const monthAgo = new Date(now.getFullYear(), now.getMonth() - 1, now.getDate()); -// const user_one = { -// user_id: 'user_one', -// client_id: 'client_one', -// connect_at: now, -// sdk: 'powersync-dart/1.6.4', -// user_agent: 'powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android', -// jwt_exp: nowAdd5minutes -// }; -// const user_two = { -// user_id: 'user_two', -// client_id: 'client_two', -// connect_at: nowLess5minutes, -// sdk: 'powersync-js/1.21.1', -// user_agent: 'powersync-js/1.21.0 powersync-web Chromium/138 linux', -// jwt_exp: nowAdd5minutes -// }; -// const user_three = { -// user_id: 'user_three', -// client_id: 'client_three', -// connect_at: yesterday, -// sdk: 'powersync-js/1.21.2', -// user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', -// disconnect_at: yesterday -// }; -// -// const user_four = { -// user_id: 'user_four', -// client_id: 'client_four', -// connect_at: now, -// sdk: 'powersync-js/1.21.4', -// user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', -// jwt_exp: nowLess5minutes -// }; -// -// const user_week = { -// user_id: 'user_week', -// client_id: 'client_week', -// connect_at: weekAgo, -// sdk: 'powersync-js/1.24.5', -// user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', -// disconnect_at: weekAgo -// }; -// -// const user_month = { -// user_id: 'user_month', -// client_id: 'client_month', -// connect_at: monthAgo, -// sdk: 'powersync-js/1.23.6', -// user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', -// disconnect_at: monthAgo -// }; -// -// const user_expired = { -// user_id: 'user_expired', -// client_id: 'client_expired', -// connect_at: monthAgo, -// sdk: 'powersync-js/1.23.7', -// user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', -// jwt_exp: monthAgo -// }; -// -// async function loadData() { -// await factory.db.sdk_report_events.insertMany([ -// user_one, -// user_two, -// user_three, -// user_four, -// user_week, -// user_month, -// user_expired -// ]); -// } -// -// function deleteData() { -// return factory.db.sdk_report_events.deleteMany(); -// } -// -// beforeAll(async () => { -// await loadData(); -// }); -// -// afterAll(async () => { -// await deleteData(); -// }); -// it('Should show connected users with start range', async () => { -// const current = await factory.listCurrentConnections({ -// range: { -// start_date: new Date( -// now.getFullYear(), -// now.getMonth(), -// now.getDate(), -// now.getHours(), -// now.getMinutes() - 1 -// ).toISOString() -// } -// }); -// expect(current).toMatchSnapshot(); -// }); -// it('Should show connected users with start range and end range', async () => { -// const current = await factory.listCurrentConnections({ -// range: { -// end_date: nowLess5minutes.toISOString(), -// start_date: new Date( -// now.getFullYear(), -// now.getMonth(), -// now.getDate(), -// now.getHours(), -// now.getMinutes() - 6 -// ).toISOString() -// } -// }); -// expect(current).toMatchSnapshot(); -// }); -// it('Should show SDK scrape data for user over the past month', async () => { -// const sdk = await factory.scrapeSdkData({ -// interval: 1, -// timeframe: 'month' -// }); -// expect(sdk).toMatchSnapshot(); -// }); -// it('Should show SDK scrape data for user over the past week', async () => { -// const sdk = await factory.scrapeSdkData({ -// interval: 1, -// timeframe: 'week' -// }); -// expect(sdk).toMatchSnapshot(); -// }); -// it('Should show SDK scrape data for user over the past day', async () => { -// const sdk = await factory.scrapeSdkData({ -// interval: 1, -// timeframe: 'day' -// }); -// expect(sdk).toMatchSnapshot(); -// }); -// -// it('Should update a sdk event if its within a day', async () => { -// const newConnectAt = new Date( -// now.getFullYear(), -// now.getMonth(), -// now.getDate(), -// now.getHours(), -// now.getMinutes() + 20 -// ); -// const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); -// await factory.reportSdkConnect({ -// sdk: user_one.sdk, -// connect_at: newConnectAt, -// jwt_exp: jwtExp, -// client_id: user_one.client_id, -// user_id: user_one.user_id, -// user_agent: user_one.user_agent -// }); -// -// const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); -// expect(sdk).toHaveLength(1); -// expect(new Date(sdk[0].connect_at)).toEqual(newConnectAt); -// expect(new Date(sdk[0].jwt_exp!)).toEqual(jwtExp); -// expect(sdk[0].disconnect_at).toBeNull(); -// const cleaned = removeVolatileFields(sdk); -// expect(cleaned).toMatchSnapshot(); -// }); -// -// it('Should update a connected sdk event and make it disconnected', async () => { -// const disconnectAt = new Date( -// now.getFullYear(), -// now.getMonth(), -// now.getDate(), -// now.getHours(), -// now.getMinutes() + 20 -// ); -// const jwtExp = new Date(disconnectAt.getFullYear(), disconnectAt.getMonth(), disconnectAt.getDate() + 1); -// -// await factory.reportSdkDisconnect({ -// disconnect_at: disconnectAt, -// jwt_exp: jwtExp, -// client_id: user_one.client_id, -// user_id: user_one.user_id, -// user_agent: user_one.user_agent -// }); -// -// const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); -// expect(sdk).toHaveLength(1); -// expect(new Date(sdk[0].disconnect_at!)).toEqual(disconnectAt); -// const cleaned = removeVolatileFields(sdk); -// expect(cleaned).toMatchSnapshot(); -// }); -// -// it('Should create a sdk event if its after a day', async () => { -// const newConnectAt = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, now.getHours()); -// const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); -// -// await factory.reportSdkConnect({ -// sdk: user_week.sdk, -// connect_at: newConnectAt, -// jwt_exp: jwtExp, -// client_id: user_week.client_id, -// user_id: user_week.user_id, -// user_agent: user_week.user_agent -// }); -// -// const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); -// expect(sdk).toHaveLength(2); -// const cleaned = removeVolatileFields(sdk); -// expect(cleaned).toMatchSnapshot(); -// }); -// -// it('Should delete rows older than specified range', async () => { -// await deleteData(); -// await loadData(); -// await factory.deleteOldSdkData({ -// interval: 1, -// timeframe: 'week' -// }); -// const sdk = await factory.scrapeSdkData({ -// interval: 1, -// timeframe: 'month' -// }); -// expect(sdk).toMatchSnapshot(); -// }); -// }); From 10a0164c40552e882fcf9962ccbfe17c5cb2c825 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 4 Aug 2025 11:36:34 +0200 Subject: [PATCH 147/169] changed the disconnect report to use connect_at time for long running connections --- .changeset/wet-berries-enjoy.md | 15 ++++++------- .../src/storage/MongoReportStorage.ts | 21 ++++++++++++------- .../sdk-report-storage.test.ts.snap | 8 +++---- .../test/src/sdk-report-storage.test.ts | 9 ++++---- .../storage/PostgresReportStorageFactory.ts | 7 +++---- .../sdk-report-storage.test.ts.snap | 8 +++---- .../test/src/sdk-report-storage.test.ts | 9 ++++---- .../src/routes/endpoints/socket-route.ts | 11 ++++------ .../src/routes/endpoints/sync-stream.ts | 11 ++++------ packages/types/src/events.ts | 2 ++ 10 files changed, 53 insertions(+), 48 deletions(-) diff --git a/.changeset/wet-berries-enjoy.md b/.changeset/wet-berries-enjoy.md index 5d07184b2..6f18739ad 100644 --- a/.changeset/wet-berries-enjoy.md +++ b/.changeset/wet-berries-enjoy.md @@ -9,13 +9,14 @@ '@powersync/service-sync-rules': minor --- -MySQL: +MySQL: + - Added schema change handling - - Except for some edge cases, the following schema changes are now handled automatically: - - Creation, renaming, dropping and truncation of tables. - - Creation and dropping of unique indexes and primary keys. - - Adding, modifying, dropping and renaming of table columns. - - If a schema change cannot handled automatically, a warning with details will be logged. - - Mismatches in table schema from the Zongji binlog listener are now handled more gracefully. +- Except for some edge cases, the following schema changes are now handled automatically: + - Creation, renaming, dropping and truncation of tables. + - Creation and dropping of unique indexes and primary keys. + - Adding, modifying, dropping and renaming of table columns. +- If a schema change cannot handled automatically, a warning with details will be logged. +- Mismatches in table schema from the Zongji binlog listener are now handled more gracefully. - Replication of wildcard tables is now supported. - Improved logging for binlog event processing. diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index cb59e0b33..31f8d2964 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -210,15 +210,22 @@ export class MongoReportStorage implements storage.ReportStorageFactory { ); } async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { - const updateFilter = this.updateDocFilter(data.user_id, data.client_id!); - await this.db.sdk_report_events.findOneAndUpdate(updateFilter, { - $set: { - disconnect_at: data.disconnect_at + const { connect_at, user_id, client_id } = data; + await this.db.sdk_report_events.findOneAndUpdate( + { + client_id, + user_id, + connect_at }, - $unset: { - jwt_exp: '' + { + $set: { + disconnect_at: data.disconnect_at + }, + $unset: { + jwt_exp: '' + } } - }); + ); } async listCurrentConnections( data: event_types.ListCurrentConnectionsRequest diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap index 7ddd94b2d..fd17f625d 100644 --- a/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap +++ b/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap @@ -168,10 +168,10 @@ exports[`SDK reporting storage > Should show connected users with start range an exports[`SDK reporting storage > Should update a connected sdk event and make it disconnected 1`] = ` [ { - "client_id": "client_one", - "sdk": "powersync-dart/1.6.4", - "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android", - "user_id": "user_one", + "client_id": "client_three", + "sdk": "powersync-js/1.21.2", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_three", }, ] `; diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts index 60c8cc502..4e4f23f85 100644 --- a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts @@ -208,12 +208,13 @@ describe('SDK reporting storage', async () => { await factory.reportSdkDisconnect({ disconnect_at: disconnectAt, jwt_exp: jwtExp, - client_id: user_one.client_id, - user_id: user_one.user_id, - user_agent: user_one.user_agent + client_id: user_three.client_id, + user_id: user_three.user_id, + user_agent: user_three.user_agent, + connect_at: user_three.connect_at }); - const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); + const sdk = await factory.db.sdk_report_events.find({ user_id: user_three.user_id }).toArray(); expect(sdk).toHaveLength(1); expect(new Date(sdk[0].disconnect_at!)).toEqual(disconnectAt); const cleaned = removeVolatileFields(sdk); diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index ca6399f43..f98ec3818 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -275,9 +275,9 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { - const { user_id, client_id, disconnect_at } = data; + const { user_id, client_id, disconnect_at, connect_at } = data; const disconnectIsoString = disconnect_at.toISOString(); - const { gte, lt } = this.updateTableFilter(); + const connectIsoString = connect_at.toISOString(); await this.db.sql` UPDATE sdk_report_events SET @@ -286,8 +286,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor WHERE user_id = ${{ type: 'varchar', value: user_id }} AND client_id = ${{ type: 'varchar', value: client_id }} - AND connect_at >= ${{ type: 1184, value: gte }} - AND connect_at < ${{ type: 1184, value: lt }}; + AND connect_at = ${{ type: 1184, value: connectIsoString }} `.execute(); } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { diff --git a/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap index 7ddd94b2d..fd17f625d 100644 --- a/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap +++ b/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap @@ -168,10 +168,10 @@ exports[`SDK reporting storage > Should show connected users with start range an exports[`SDK reporting storage > Should update a connected sdk event and make it disconnected 1`] = ` [ { - "client_id": "client_one", - "sdk": "powersync-dart/1.6.4", - "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android", - "user_id": "user_one", + "client_id": "client_three", + "sdk": "powersync-js/1.21.2", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_three", }, ] `; diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index a5666f277..28f67aa3f 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -289,13 +289,14 @@ describe('SDK reporting storage', async () => { await factory.reportSdkDisconnect({ disconnect_at: disconnectAt, jwt_exp: jwtExp, - client_id: user_one.client_id, - user_id: user_one.user_id, - user_agent: user_one.user_agent + client_id: user_three.client_id, + user_id: user_three.user_id, + user_agent: user_three.user_agent, + connect_at: yesterday }); const sdk = await factory.db - .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_one.user_id }}`.rows(); + .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_three.user_id }}`.rows(); expect(sdk).toHaveLength(1); expect(new Date(sdk[0].disconnect_at!).toISOString()).toEqual(disconnectAt.toISOString()); const cleaned = removeVolatileFields(sdk); diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index f802044c5..163d947ba 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -14,6 +14,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => { const { service_context, logger } = context; const { routerEngine, metricsEngine, syncContext } = service_context; + const streamStart = Date.now(); logger.defaultMeta = { ...logger.defaultMeta, @@ -26,11 +27,10 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => client_id: params.client_id, user_id: context.user_id!, user_agent: context.user_agent, - jwt_exp: context.token_payload?.exp ? new Date(context.token_payload.exp * 1000) : undefined + jwt_exp: context.token_payload?.exp ? new Date(context.token_payload.exp * 1000) : undefined, + connect_at: new Date(streamStart) }; - const streamStart = Date.now(); - // Best effort guess on why the stream was closed. // We use the `??=` operator everywhere, so that we catch the first relevant // event, which is usually the most specific. @@ -93,10 +93,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => }); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, { - ...sdkData, - connect_at: new Date(streamStart) - }); + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, sdkData); const tracker = new sync.RequestTracker(metricsEngine); try { for await (const data of sync.streamResponse({ diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index f1e1cb7ad..4635ae1ef 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -1,4 +1,4 @@ -import { ErrorCode, errors, logger, router, schema } from '@powersync/lib-services-framework'; +import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework'; import { RequestParameters } from '@powersync/service-sync-rules'; import { Readable } from 'stream'; import Negotiator from 'negotiator'; @@ -46,7 +46,8 @@ export const syncStreamed = routeDefinition({ client_id: clientId, user_id: payload.context.user_id!, user_agent: userAgent as string, - jwt_exp: token_payload?.exp ? new Date(token_payload?.exp * 1000) : undefined + jwt_exp: token_payload?.exp ? new Date(token_payload?.exp * 1000) : undefined, + connect_at: new Date(streamStart) }; if (routerEngine.closed) { @@ -93,11 +94,7 @@ export const syncStreamed = routeDefinition({ objectMode: false, highWaterMark: 16 * 1024 }); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, { - ...sdkData, - connect_at: new Date(streamStart) - }); - + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, sdkData); // Best effort guess on why the stream was closed. // We use the `??=` operator everywhere, so that we catch the first relevant diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index eb6918949..fb1beb312 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -22,6 +22,7 @@ export type SdkUserData = { user_id: string; user_agent?: string; jwt_exp?: Date; + connect_at: Date; }; export type DeleteOldSdkData = { @@ -40,6 +41,7 @@ export type SdkConnectBucketData = { export type SdkDisconnectEventData = { disconnect_at: Date; + connect_at: Date; } & SdkUserData; export type SdkConnectDocument = { From 51661d885f3c3be28c66444ede1c5c9bbab2adcf Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 11 Aug 2025 16:48:39 +0200 Subject: [PATCH 148/169] fixed connected at filter --- .../src/storage/MongoReportStorage.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 31f8d2964..571340879 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -98,8 +98,10 @@ export class MongoReportStorage implements storage.ReportStorageFactory { const endDate = data.range?.end_date ? new Date(data.range.end_date) : new Date(); const startDate = new Date(range.start_date); return { - $lte: endDate, - $gt: startDate + connect_at: { + $lte: endDate, + $gt: startDate + } }; } @@ -237,7 +239,7 @@ export class MongoReportStorage implements storage.ReportStorageFactory { $match: { disconnect_at: { $exists: false }, jwt_exp: { $gt: new Date() }, - connect_at: timeframeFilter + ...timeframeFilter } }, this.sdkFacetPipeline(), From 1a6ad4042423f7a2e0ef3f7a4f060d33fbc22bf5 Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 12 Aug 2025 16:25:02 +0200 Subject: [PATCH 149/169] cclean up some code for pr --- modules/module-core/package.json | 2 +- modules/module-mongodb/package.json | 2 +- modules/module-mysql/package.json | 2 +- modules/module-postgres/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/module-core/package.json b/modules/module-core/package.json index 3b6913c50..16d93bee0 100644 --- a/modules/module-core/package.json +++ b/modules/module-core/package.json @@ -30,8 +30,8 @@ "dependencies": { "@powersync/lib-services-framework": "workspace:*", "@powersync/service-core": "workspace:*", - "@powersync/service-types": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", + "@powersync/service-types": "workspace:*", "fastify": "4.23.2", "@fastify/cors": "8.4.1" }, diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index c251255bd..87812cfdf 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -31,9 +31,9 @@ "@powersync/lib-service-mongodb": "workspace:*", "@powersync/lib-services-framework": "workspace:*", "@powersync/service-core": "workspace:*", - "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", "bson": "^6.10.3", "ts-codec": "^1.3.0", "uuid": "^11.1.0" diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index c7d17036b..3d11f5a25 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -30,8 +30,8 @@ "dependencies": { "@powersync/lib-services-framework": "workspace:*", "@powersync/service-core": "workspace:*", - "@powersync/service-types": "workspace:*", "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/mysql-zongji": "^0.4.0", "async": "^3.2.4", diff --git a/modules/module-postgres/package.json b/modules/module-postgres/package.json index 6707da0f2..a5003a4c8 100644 --- a/modules/module-postgres/package.json +++ b/modules/module-postgres/package.json @@ -31,10 +31,10 @@ "@powersync/lib-service-postgres": "workspace:*", "@powersync/lib-services-framework": "workspace:*", "@powersync/service-core": "workspace:*", - "@powersync/service-types": "workspace:*", "@powersync/service-jpgwire": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", "jose": "^4.15.1", "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87", "semver": "^7.5.4", From 0fad4668820da11c2e7cfa417da5fc5f8be3ff5c Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 12 Aug 2025 16:27:50 +0200 Subject: [PATCH 150/169] version bump --- .changeset/honest-sloths-smoke.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/honest-sloths-smoke.md diff --git a/.changeset/honest-sloths-smoke.md b/.changeset/honest-sloths-smoke.md new file mode 100644 index 000000000..4e50441a1 --- /dev/null +++ b/.changeset/honest-sloths-smoke.md @@ -0,0 +1,8 @@ +--- +'@powersync/service-module-postgres-storage': patch +'@powersync/service-module-mongodb-storage': patch +'@powersync/service-core': patch +'@powersync/service-types': patch +--- + +Added sdk reporting to storage From 4ac4ed33236f9f38b3ecc6217e8fa610e302d576 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 13 Aug 2025 09:47:32 +0200 Subject: [PATCH 151/169] chnages to date ranges on scrapes, simplified --- .../src/storage/MongoReportStorage.ts | 74 ++---------------- .../sdk-report-storage.test.ts.snap | 14 +++- .../test/src/sdk-report-storage.test.ts | 20 ++--- .../storage/PostgresReportStorageFactory.ts | 76 ++----------------- .../sdk-report-storage.test.ts.snap | 14 +++- .../test/src/sdk-report-storage.test.ts | 20 ++--- packages/types/src/events.ts | 7 +- 7 files changed, 60 insertions(+), 165 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 571340879..4fb16c1e0 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -100,93 +100,31 @@ export class MongoReportStorage implements storage.ReportStorageFactory { return { connect_at: { $lte: endDate, - $gt: startDate + $gte: startDate } }; } - private timeFrameQuery(timeframe: event_types.TimeFrames, interval: number = 1): mongo.Filter { - const { year, month, today, parsedDate } = this.parseJsDate(new Date()); - switch (timeframe) { - case 'month': { - return { $lte: parsedDate, $gt: new Date(year, parsedDate.getMonth() - interval) }; - } - case 'week': { - const weekStartDate = new Date(parsedDate); - weekStartDate.setDate(weekStartDate.getDate() - 6 * interval); - const weekStart = this.parseJsDate(weekStartDate); - return { - $lte: parsedDate, - $gt: new Date(weekStart.year, weekStart.month, weekStart.today) - }; - } - case 'hour': { - // Get the last hour from the current time - const previousHour = parsedDate.getHours() - interval; - return { - $gt: new Date(year, month, today, previousHour), - $lte: new Date(year, month, today, parsedDate.getHours()) - }; - } - default: { - return { - $lte: parsedDate, - $gt: new Date(year, month, today - interval) - }; - } - } - } - - private timeFrameDeleteQuery(timeframe: event_types.TimeFrames, interval: number = 1): mongo.Filter { - const { year, month, today, parsedDate } = this.parseJsDate(new Date()); - switch (timeframe) { - case 'month': { - return { $lt: new Date(year, parsedDate.getMonth() - interval) }; - } - case 'week': { - const weekStartDate = new Date(parsedDate); - weekStartDate.setDate(weekStartDate.getDate() - 6 * interval); - const { month, year, today } = this.parseJsDate(weekStartDate); - return { - $lt: new Date(year, month, today) - }; - } - case 'hour': { - const previousHour = parsedDate.getHours() - interval; - return { - $lt: new Date(year, month, today, previousHour) - }; - } - default: { - return { - $lt: new Date(year, month, today - interval) - }; - } - } - } - async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise { - const { interval, timeframe } = data; - const timeframeFilter = this.timeFrameDeleteQuery(timeframe, interval); + const { date } = data; const result = await this.db.sdk_report_events.deleteMany({ - connect_at: timeframeFilter, + connect_at: { $lt: date }, $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }] }); if (result.deletedCount > 0) { logger.info( - `TTL ${interval}/${timeframe}: ${result.deletedCount} MongoDB documents have been removed from sdk_report_events.` + `TTL from ${date.toISOString()}: ${result.deletedCount} MongoDB documents have been removed from sdk_report_events.` ); } } async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { - const { interval, timeframe } = data; - const timeframeFilter = this.timeFrameQuery(timeframe, interval); + const { start, end } = data; const result = await this.db.sdk_report_events .aggregate([ { $match: { - connect_at: timeframeFilter + connect_at: { $lte: end, $gte: start } } }, this.sdkFacetPipeline(), diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap index fd17f625d..77ab95509 100644 --- a/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap +++ b/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap @@ -40,8 +40,13 @@ exports[`SDK reporting storage > Should delete rows older than specified range 1 "sdk": "powersync-js/1.21.4", "users": 1, }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, + }, ], - "users": 4, + "users": 5, } `; @@ -134,8 +139,13 @@ exports[`SDK reporting storage > Should show SDK scrape data for user over the p "sdk": "powersync-js/1.21.4", "users": 1, }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, + }, ], - "users": 4, + "users": 5, } `; diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts index 4e4f23f85..9d8f8e601 100644 --- a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts @@ -30,6 +30,7 @@ describe('SDK reporting storage', async () => { now.getHours(), now.getMinutes() - 5 ); + const dayAgo = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1, now.getHours()); const yesterday = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1); const weekAgo = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7); const monthAgo = new Date(now.getFullYear(), now.getMonth() - 1, now.getDate()); @@ -148,22 +149,22 @@ describe('SDK reporting storage', async () => { }); it('Should show SDK scrape data for user over the past month', async () => { const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'month' + start: monthAgo, + end: now }); expect(sdk).toMatchSnapshot(); }); it('Should show SDK scrape data for user over the past week', async () => { const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'week' + start: weekAgo, + end: now }); expect(sdk).toMatchSnapshot(); }); it('Should show SDK scrape data for user over the past day', async () => { const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'day' + start: dayAgo, + end: now }); expect(sdk).toMatchSnapshot(); }); @@ -244,12 +245,11 @@ describe('SDK reporting storage', async () => { await deleteData(); await loadData(); await factory.deleteOldSdkData({ - interval: 1, - timeframe: 'week' + date: weekAgo }); const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'month' + start: monthAgo, + end: now }); expect(sdk).toMatchSnapshot(); }); diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index f98ec3818..3f0f0d71e 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -49,66 +49,6 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor parsedDate: date }; } - private timeFrameQuery(timeframe: event_types.TimeFrames, interval: number = 1) { - const { year, month, today, parsedDate } = this.parseJsDate(new Date()); - const parsedIsoString = parsedDate.toISOString(); - switch (timeframe) { - case 'month': { - return { lt: parsedIsoString, gt: new Date(year, parsedDate.getMonth() - interval - 1).toISOString() }; - } - case 'week': { - const weekStartDate = new Date(parsedDate); - weekStartDate.setDate(weekStartDate.getDate() - 6 * interval); - const weekStart = this.parseJsDate(weekStartDate); - return { - lt: parsedIsoString, - gt: new Date(weekStart.year, weekStart.month, weekStart.today).toISOString() - }; - } - case 'hour': { - // Get the last hour from the current time - const previousHour = parsedDate.getHours() - interval; - return { - lt: new Date(year, month, today, parsedDate.getHours()).toISOString(), - gt: new Date(year, month, today, previousHour).toISOString() - }; - } - default: { - return { - lt: parsedIsoString, - gt: new Date(year, month, today - interval).toISOString() - }; - } - } - } - - private timeFrameDeleteQuery(timeframe: event_types.TimeFrames, interval: number = 1, test_date?: Date) { - const { year, month, today, parsedDate } = test_date ? this.parseJsDate(test_date) : this.parseJsDate(new Date()); - switch (timeframe) { - case 'month': { - return { lt: new Date(year, parsedDate.getMonth() - interval).toISOString() }; - } - case 'week': { - const weekStartDate = new Date(parsedDate); - weekStartDate.setDate(weekStartDate.getDate() - 6 * interval); - const { month, year, today } = this.parseJsDate(weekStartDate); - return { - lt: new Date(year, month, today).toISOString() - }; - } - case 'hour': { - const previousHour = parsedDate.getHours() - interval; - return { - lt: new Date(year, month, today, previousHour).toISOString() - }; - } - default: { - return { - lt: new Date(year, month, today - interval).toISOString() - }; - } - } - } private mapListCurrentConnectionsResponse(result: SdkReportingDecoded | null): ListCurrentConnections { if (!result) { @@ -183,7 +123,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor WHERE disconnect_at IS NULL AND jwt_exp > NOW() - AND connect_at > ${{ type: 1184, value: gt }} + AND connect_at >= ${{ type: 1184, value: gt }} AND connect_at <= ${{ type: 1184, value: lt }} ), unique_users AS ( @@ -295,8 +235,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor } async scrapeSdkData(data: ScrapeSdkDataRequest): Promise { - const { timeframe, interval } = data; - const { lt, gt } = this.timeFrameQuery(timeframe, interval); + const { start, end } = data; const result = await this.db.sql` WITH filtered AS ( @@ -305,8 +244,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor FROM sdk_report_events WHERE - connect_at > ${{ type: 1184, value: gt }} - AND connect_at <= ${{ type: 1184, value: lt }} + connect_at >= ${{ type: 1184, value: start.toISOString() }} + AND connect_at <= ${{ type: 1184, value: end.toISOString() }} ), unique_users AS ( SELECT @@ -343,12 +282,11 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor return this.mapListCurrentConnectionsResponse(result); } async deleteOldSdkData(data: DeleteOldSdkData): Promise { - const { timeframe, interval } = data; - const { lt } = this.timeFrameDeleteQuery(timeframe, interval); + const { date } = data; const result = await this.db.sql` DELETE FROM sdk_report_events WHERE - connect_at < ${{ type: 1184, value: lt }} + connect_at < ${{ type: 1184, value: date.toISOString() }} AND ( disconnect_at IS NOT NULL OR ( @@ -360,7 +298,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor const deletedRows = toInteger(result.results[1].status.split(' ')[1] || '0'); if (deletedRows > 0) { logger.info( - `TTL ${interval}/${timeframe}: ${deletedRows} PostgresSQL rows have been removed from sdk_report_events.` + `TTL from ${date.toISOString()}: ${deletedRows} PostgresSQL rows have been removed from sdk_report_events.` ); } } diff --git a/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap index fd17f625d..77ab95509 100644 --- a/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap +++ b/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap @@ -40,8 +40,13 @@ exports[`SDK reporting storage > Should delete rows older than specified range 1 "sdk": "powersync-js/1.21.4", "users": 1, }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, + }, ], - "users": 4, + "users": 5, } `; @@ -134,8 +139,13 @@ exports[`SDK reporting storage > Should show SDK scrape data for user over the p "sdk": "powersync-js/1.21.4", "users": 1, }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, + }, ], - "users": 4, + "users": 5, } `; diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index 28f67aa3f..a30d98b88 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -28,6 +28,7 @@ describe('SDK reporting storage', async () => { now.getHours(), now.getMinutes() - 5 ); + const dayAgo = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1, now.getHours()); const yesterday = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1); const weekAgo = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7); const monthAgo = new Date(now.getFullYear(), now.getMonth() - 1, now.getDate()); @@ -228,22 +229,22 @@ describe('SDK reporting storage', async () => { }); it('Should show SDK scrape data for user over the past month', async () => { const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'month' + start: monthAgo, + end: now }); expect(sdk).toMatchSnapshot(); }); it('Should show SDK scrape data for user over the past week', async () => { const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'week' + start: weekAgo, + end: now }); expect(sdk).toMatchSnapshot(); }); it('Should show SDK scrape data for user over the past day', async () => { const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'day' + start: dayAgo, + end: now }); expect(sdk).toMatchSnapshot(); }); @@ -327,12 +328,11 @@ describe('SDK reporting storage', async () => { await deleteData(); await loadData(); await factory.deleteOldSdkData({ - interval: 1, - timeframe: 'week' + date: weekAgo }); const sdk = await factory.scrapeSdkData({ - interval: 1, - timeframe: 'month' + start: monthAgo, + end: now }); expect(sdk).toMatchSnapshot(); }); diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index fb1beb312..3033397e0 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -26,8 +26,7 @@ export type SdkUserData = { }; export type DeleteOldSdkData = { - interval: number; - timeframe: TimeFrames; + date: Date; }; export type SdkConnectEventData = { @@ -65,8 +64,8 @@ export type ListCurrentConnections = { }; export type ScrapeSdkDataRequest = { - timeframe: TimeFrames; - interval?: number; + start: Date; + end: Date; }; export type ListCurrentConnectionsRequest = { From 877ce668b150abfff5710bb77ae3cdc234535ea7 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 13 Aug 2025 10:02:23 +0200 Subject: [PATCH 152/169] fixed list current connections payload --- .../src/storage/MongoReportStorage.ts | 4 ++-- .../test/src/sdk-report-storage.test.ts | 6 +++--- .../src/storage/PostgresReportStorageFactory.ts | 4 ++-- .../test/src/sdk-report-storage.test.ts | 6 +++--- packages/types/src/events.ts | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 4fb16c1e0..6fdcbbc86 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -95,8 +95,8 @@ export class MongoReportStorage implements storage.ReportStorageFactory { if (!range) { return undefined; } - const endDate = data.range?.end_date ? new Date(data.range.end_date) : new Date(); - const startDate = new Date(range.start_date); + const endDate = data.range?.end ? new Date(data.range.end) : new Date(); + const startDate = new Date(range.start); return { connect_at: { $lte: endDate, diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts index 9d8f8e601..a3dc5be21 100644 --- a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts @@ -121,7 +121,7 @@ describe('SDK reporting storage', async () => { it('Should show connected users with start range', async () => { const current = await factory.listCurrentConnections({ range: { - start_date: new Date( + start: new Date( now.getFullYear(), now.getMonth(), now.getDate(), @@ -135,8 +135,8 @@ describe('SDK reporting storage', async () => { it('Should show connected users with start range and end range', async () => { const current = await factory.listCurrentConnections({ range: { - end_date: nowLess5minutes.toISOString(), - start_date: new Date( + end: nowLess5minutes.toISOString(), + start: new Date( now.getFullYear(), now.getMonth(), now.getDate(), diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 3f0f0d71e..3024579dd 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -109,8 +109,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorageFactor .decoded(SdkReporting) .first(); } - const endDate = data.range?.end_date ? new Date(data.range.end_date) : new Date(); - const startDate = new Date(range.start_date); + const endDate = data.range?.end ? new Date(data.range.end) : new Date(); + const startDate = new Date(range.start); const lt = endDate.toISOString(); const gt = startDate.toISOString(); return await this.db.sql` diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index a30d98b88..32f17bad9 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -201,7 +201,7 @@ describe('SDK reporting storage', async () => { it('Should show connected users with start range', async () => { const current = await factory.listCurrentConnections({ range: { - start_date: new Date( + start: new Date( now.getFullYear(), now.getMonth(), now.getDate(), @@ -215,8 +215,8 @@ describe('SDK reporting storage', async () => { it('Should show connected users with start range and end range', async () => { const current = await factory.listCurrentConnections({ range: { - end_date: nowLess5minutes.toISOString(), - start_date: new Date( + end: nowLess5minutes.toISOString(), + start: new Date( now.getFullYear(), now.getMonth(), now.getDate(), diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 3033397e0..e91bbfe8f 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -70,7 +70,7 @@ export type ScrapeSdkDataRequest = { export type ListCurrentConnectionsRequest = { range?: { - start_date: string; - end_date?: string; + start: string; + end?: string; }; }; From 29a084581a35666877235dc6eaf046cdbe5cea38 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 14 Aug 2025 15:12:56 +0200 Subject: [PATCH 153/169] fix migration indexes on down --- .../db/migrations/1752661449910-sdk-reporting.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts index 9ad09e9df..55f2d82f1 100644 --- a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts +++ b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts @@ -51,14 +51,17 @@ export const down: migrations.PowerSyncMigrationFunction = async (context) => { const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig); try { - if (await db.write_checkpoints.indexExists('connect_at')) { - await db.write_checkpoints.dropIndex('connect_at'); + if (await db.sdk_report_events.indexExists('sdk_list_index')) { + await db.sdk_report_events.dropIndex('sdk_list_index'); } - if (await db.custom_write_checkpoints.indexExists('user_id')) { - await db.custom_write_checkpoints.dropIndex('user_id'); + if (await db.sdk_report_events.indexExists('sdk_user_id_index')) { + await db.sdk_report_events.dropIndex('sdk_user_id_index'); } - if (await db.custom_write_checkpoints.indexExists('client_id')) { - await db.custom_write_checkpoints.dropIndex('client_id'); + if (await db.sdk_report_events.indexExists('sdk_client_id_index')) { + await db.sdk_report_events.dropIndex('sdk_client_id_index'); + } + if (await db.sdk_report_events.indexExists('sdk_index')) { + await db.sdk_report_events.dropIndex('sdk_index'); } await db.db.dropCollection('sdk_report_events'); } finally { From ea0c2122525dfc79f3058623985cac00eaab8631 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 14 Aug 2025 15:25:02 +0200 Subject: [PATCH 154/169] model document naming --- .../module-mongodb-storage/src/storage/implementation/db.ts | 4 ++-- .../src/storage/implementation/models.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index ae2c1fc91..275c4fd8a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -12,7 +12,7 @@ import { CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, - SdkConnectDocument, + SdkConnectEventDocument, SourceTableDocument, SyncRuleDocument, WriteCheckpointDocument @@ -38,7 +38,7 @@ export class PowerSyncMongo { readonly locks: mongo.Collection; readonly bucket_state: mongo.Collection; readonly checkpoint_events: mongo.Collection; - readonly sdk_report_events: mongo.Collection; + readonly sdk_report_events: mongo.Collection; readonly client: mongo.MongoClient; readonly db: mongo.Db; diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index a054e287e..eadff6b0d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -221,4 +221,4 @@ export interface InstanceDocument { _id: string; } -export interface SdkConnectDocument extends event_types.SdkConnectDocument {} +export interface SdkConnectEventDocument extends event_types.SdkConnectDocument {} From c1b6a0c2f7095888814c6240c3f561105f47b171 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 14 Aug 2025 15:29:57 +0200 Subject: [PATCH 155/169] moved abstarct methods to the top --- .../src/storage/MongoReportStorage.ts | 170 +++++++++--------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 6fdcbbc86..f176402c2 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -12,6 +12,91 @@ export class MongoReportStorage implements storage.ReportStorageFactory { this.client = db.client; this.db = db; } + async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise { + const { date } = data; + const result = await this.db.sdk_report_events.deleteMany({ + connect_at: { $lt: date }, + $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }] + }); + if (result.deletedCount > 0) { + logger.info( + `TTL from ${date.toISOString()}: ${result.deletedCount} MongoDB documents have been removed from sdk_report_events.` + ); + } + } + + async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { + const { start, end } = data; + const result = await this.db.sdk_report_events + .aggregate([ + { + $match: { + connect_at: { $lte: end, $gte: start } + } + }, + this.sdkFacetPipeline(), + this.sdkProjectPipeline() + ]) + .toArray(); + return result[0]; + } + + async reportSdkConnect(data: event_types.SdkConnectBucketData): Promise { + const updateFilter = this.updateDocFilter(data.user_id, data.client_id!); + await this.db.sdk_report_events.findOneAndUpdate( + updateFilter, + { + $set: data, + $unset: { + disconnect_at: '' + } + }, + { + upsert: true + } + ); + } + async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { + const { connect_at, user_id, client_id } = data; + await this.db.sdk_report_events.findOneAndUpdate( + { + client_id, + user_id, + connect_at + }, + { + $set: { + disconnect_at: data.disconnect_at + }, + $unset: { + jwt_exp: '' + } + } + ); + } + async listCurrentConnections( + data: event_types.ListCurrentConnectionsRequest + ): Promise { + const timeframeFilter = this.listConnectionsDateRange(data); + const result = await this.db.sdk_report_events + .aggregate([ + { + $match: { + disconnect_at: { $exists: false }, + jwt_exp: { $gt: new Date() }, + ...timeframeFilter + } + }, + this.sdkFacetPipeline(), + this.sdkProjectPipeline() + ]) + .toArray(); + return result[0]; + } + + async [Symbol.asyncDispose]() { + // No-op + } private parseJsDate(date: Date) { const year = date.getFullYear(); @@ -105,89 +190,4 @@ export class MongoReportStorage implements storage.ReportStorageFactory { }; } - async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise { - const { date } = data; - const result = await this.db.sdk_report_events.deleteMany({ - connect_at: { $lt: date }, - $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }] - }); - if (result.deletedCount > 0) { - logger.info( - `TTL from ${date.toISOString()}: ${result.deletedCount} MongoDB documents have been removed from sdk_report_events.` - ); - } - } - - async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { - const { start, end } = data; - const result = await this.db.sdk_report_events - .aggregate([ - { - $match: { - connect_at: { $lte: end, $gte: start } - } - }, - this.sdkFacetPipeline(), - this.sdkProjectPipeline() - ]) - .toArray(); - return result[0]; - } - - async reportSdkConnect(data: event_types.SdkConnectBucketData): Promise { - const updateFilter = this.updateDocFilter(data.user_id, data.client_id!); - await this.db.sdk_report_events.findOneAndUpdate( - updateFilter, - { - $set: data, - $unset: { - disconnect_at: '' - } - }, - { - upsert: true - } - ); - } - async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { - const { connect_at, user_id, client_id } = data; - await this.db.sdk_report_events.findOneAndUpdate( - { - client_id, - user_id, - connect_at - }, - { - $set: { - disconnect_at: data.disconnect_at - }, - $unset: { - jwt_exp: '' - } - } - ); - } - async listCurrentConnections( - data: event_types.ListCurrentConnectionsRequest - ): Promise { - const timeframeFilter = this.listConnectionsDateRange(data); - const result = await this.db.sdk_report_events - .aggregate([ - { - $match: { - disconnect_at: { $exists: false }, - jwt_exp: { $gt: new Date() }, - ...timeframeFilter - } - }, - this.sdkFacetPipeline(), - this.sdkProjectPipeline() - ]) - .toArray(); - return result[0]; - } - - async [Symbol.asyncDispose]() { - // No-op - } } From 5a9c5cefe7dccdb6921867f16346bff2de3c9fc2 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 14 Aug 2025 15:36:44 +0200 Subject: [PATCH 156/169] PR review chnages --- .../src/storage/MongoReportStorage.ts | 3 +-- .../src/storage/implementation/MongoStorageProvider.ts | 2 +- .../src/storage/PostgresReportStorageFactory.ts | 2 +- .../src/storage/PostgresStorageProvider.ts | 2 +- .../storage/{ReportStorageFactory.ts => ReportStorage.ts} | 2 +- packages/service-core/src/storage/StorageEngine.ts | 6 +++--- packages/service-core/src/storage/StorageProvider.ts | 6 +++--- packages/service-core/src/storage/storage-index.ts | 2 +- 8 files changed, 12 insertions(+), 13 deletions(-) rename packages/service-core/src/storage/{ReportStorageFactory.ts => ReportStorage.ts} (88%) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index f176402c2..c3740e16b 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -4,7 +4,7 @@ import { event_types } from '@powersync/service-types'; import { PowerSyncMongo } from './implementation/db.js'; import { logger } from '@powersync/lib-services-framework'; -export class MongoReportStorage implements storage.ReportStorageFactory { +export class MongoReportStorage implements storage.ReportStorage { private readonly client: mongo.MongoClient; public readonly db: PowerSyncMongo; @@ -189,5 +189,4 @@ export class MongoReportStorage implements storage.ReportStorageFactory { } }; } - } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts index 42e211303..4ae2055b7 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoStorageProvider.ts @@ -6,7 +6,7 @@ import { MongoBucketStorage } from '../MongoBucketStorage.js'; import { PowerSyncMongo } from './db.js'; import { MongoReportStorage } from '../MongoReportStorage.js'; -export class MongoStorageProvider implements storage.BucketStorageProvider { +export class MongoStorageProvider implements storage.StorageProvider { get type() { return lib_mongo.MONGO_CONNECTION_TYPE; } diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 3024579dd..75005a838 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -22,7 +22,7 @@ export type PostgresReportStorageOptions = { config: NormalizedPostgresStorageConfig; }; -export class PostgresReportStorageFactory implements storage.ReportStorageFactory { +export class PostgresReportStorageFactory implements storage.ReportStorage { readonly db: lib_postgres.DatabaseClient; constructor(protected options: PostgresReportStorageOptions) { this.db = new lib_postgres.DatabaseClient({ diff --git a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts index 749692ff4..1fbddb4cd 100644 --- a/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts +++ b/modules/module-postgres-storage/src/storage/PostgresStorageProvider.ts @@ -7,7 +7,7 @@ import { dropTables } from '../utils/db.js'; import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; import { PostgresReportStorageFactory } from './PostgresReportStorageFactory.js'; -export class PostgresStorageProvider implements storage.BucketStorageProvider { +export class PostgresStorageProvider implements storage.StorageProvider { get type() { return lib_postgres.POSTGRES_CONNECTION_TYPE; } diff --git a/packages/service-core/src/storage/ReportStorageFactory.ts b/packages/service-core/src/storage/ReportStorage.ts similarity index 88% rename from packages/service-core/src/storage/ReportStorageFactory.ts rename to packages/service-core/src/storage/ReportStorage.ts index c28db85d6..1b8a9ae74 100644 --- a/packages/service-core/src/storage/ReportStorageFactory.ts +++ b/packages/service-core/src/storage/ReportStorage.ts @@ -1,6 +1,6 @@ import { event_types } from '@powersync/service-types'; -export interface ReportStorageFactory extends AsyncDisposable { +export interface ReportStorage extends AsyncDisposable { reportSdkConnect(data: event_types.SdkConnectBucketData): Promise; reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise; listCurrentConnections(data: event_types.ListCurrentConnectionsRequest): Promise; diff --git a/packages/service-core/src/storage/StorageEngine.ts b/packages/service-core/src/storage/StorageEngine.ts index 62fcf11f1..a53819ccb 100644 --- a/packages/service-core/src/storage/StorageEngine.ts +++ b/packages/service-core/src/storage/StorageEngine.ts @@ -1,7 +1,7 @@ import { BaseObserver, logger, ServiceError } from '@powersync/lib-services-framework'; import { ResolvedPowerSyncConfig } from '../util/util-index.js'; import { BucketStorageFactory } from './BucketStorageFactory.js'; -import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js'; +import { ActiveStorage, StorageProvider } from './StorageProvider.js'; export type StorageEngineOptions = { configuration: ResolvedPowerSyncConfig; @@ -14,7 +14,7 @@ export interface StorageEngineListener { export class StorageEngine extends BaseObserver { // TODO: This will need to revisited when we actually support multiple storage providers. - private storageProviders: Map = new Map(); + private storageProviders: Map = new Map(); private currentActiveStorage: ActiveStorage | null = null; constructor(private options: StorageEngineOptions) { @@ -37,7 +37,7 @@ export class StorageEngine extends BaseObserver { * Register a provider which generates a {@link BucketStorageFactory} * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig} */ - registerProvider(provider: BucketStorageProvider) { + registerProvider(provider: StorageProvider) { this.storageProviders.set(provider.type, provider); } diff --git a/packages/service-core/src/storage/StorageProvider.ts b/packages/service-core/src/storage/StorageProvider.ts index f813502fa..6a32e45d2 100644 --- a/packages/service-core/src/storage/StorageProvider.ts +++ b/packages/service-core/src/storage/StorageProvider.ts @@ -1,12 +1,12 @@ import { ServiceError } from '@powersync/lib-services-framework'; import * as util from '../util/util-index.js'; import { BucketStorageFactory } from './BucketStorageFactory.js'; -import { ReportStorageFactory } from './ReportStorageFactory.js'; +import { ReportStorage } from './ReportStorage.js'; export interface ActiveStorage { storage: BucketStorageFactory; // TODO: REMOVE THE NULL ONCE POSTGRES HAS BEEN IMPLEMENTED THIS IS JUST SO I CAN TEST MONGO - reportStorage: ReportStorageFactory | null; + reportStorage: ReportStorage | null; shutDown(): Promise; /** @@ -25,7 +25,7 @@ export interface GetStorageOptions { /** * Represents a provider that can create a storage instance for a specific storage type from configuration. */ -export interface BucketStorageProvider { +export interface StorageProvider { /** * The storage type that this provider provides. * The type should match the `type` field in the config. diff --git a/packages/service-core/src/storage/storage-index.ts b/packages/service-core/src/storage/storage-index.ts index 5e2cb99d5..b83a2fb2f 100644 --- a/packages/service-core/src/storage/storage-index.ts +++ b/packages/service-core/src/storage/storage-index.ts @@ -13,4 +13,4 @@ export * from './BucketStorageBatch.js'; export * from './SyncRulesBucketStorage.js'; export * from './PersistedSyncRulesContent.js'; export * from './ReplicationLock.js'; -export * from './ReportStorageFactory.js'; +export * from './ReportStorage.js'; From 4f0c77dd07e061d202624f9b1c3e33310ee00c4a Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 14 Aug 2025 15:52:24 +0200 Subject: [PATCH 157/169] PR review changes --- packages/service-core/src/routes/endpoints/sync-stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 4635ae1ef..6978190ae 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -77,6 +77,7 @@ export const syncStreamed = routeDefinition({ const tracker = new sync.RequestTracker(metricsEngine); try { metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); + service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, sdkData); const syncLines = sync.streamResponse({ syncContext: syncContext, bucketStorage, @@ -94,7 +95,6 @@ export const syncStreamed = routeDefinition({ objectMode: false, highWaterMark: 16 * 1024 }); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, sdkData); // Best effort guess on why the stream was closed. // We use the `??=` operator everywhere, so that we catch the first relevant From 8167d341386dc9ff97a927d8c2f38e8de3ca5a5a Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 14 Aug 2025 17:17:17 +0200 Subject: [PATCH 158/169] emitter PR chnages --- .../emitters/{EmitterEngine.ts => EventsEngine.ts} | 5 ++--- .../service-core/src/emitters/emitter-interfaces.ts | 12 ------------ packages/service-core/src/system/ServiceContext.ts | 11 +++++++---- 3 files changed, 9 insertions(+), 19 deletions(-) rename packages/service-core/src/emitters/{EmitterEngine.ts => EventsEngine.ts} (86%) delete mode 100644 packages/service-core/src/emitters/emitter-interfaces.ts diff --git a/packages/service-core/src/emitters/EmitterEngine.ts b/packages/service-core/src/emitters/EventsEngine.ts similarity index 86% rename from packages/service-core/src/emitters/EmitterEngine.ts rename to packages/service-core/src/emitters/EventsEngine.ts index c5e7b6009..22b747de9 100644 --- a/packages/service-core/src/emitters/EmitterEngine.ts +++ b/packages/service-core/src/emitters/EventsEngine.ts @@ -1,11 +1,10 @@ import EventEmitter from 'node:events'; import { logger } from '@powersync/lib-services-framework'; import { event_types } from '@powersync/service-types'; -import { BaseEmitterEngine } from './emitter-interfaces.js'; -export class EmitterEngine implements BaseEmitterEngine { +export class EventsEngine { private emitter: EventEmitter; - events: Set = new Set(); + private events: Set = new Set(); constructor() { this.emitter = new EventEmitter({ captureRejections: true }); this.emitter.on('error', (error: Error) => { diff --git a/packages/service-core/src/emitters/emitter-interfaces.ts b/packages/service-core/src/emitters/emitter-interfaces.ts deleted file mode 100644 index 0b75e9c5f..000000000 --- a/packages/service-core/src/emitters/emitter-interfaces.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { event_types } from '@powersync/service-types'; - -export interface BaseEmitterEngine { - listEvents: event_types.EmitterEngineEvents[]; - countListeners(eventName: event_types.EmitterEngineEvents): number; - emit( - event: event_types.EmitterEngineEvents, - data: event_types.SubscribeEvents[K] - ): void; - subscribe(event: event_types.EmitterEvent): void; - shutDown(): void; -} diff --git a/packages/service-core/src/system/ServiceContext.ts b/packages/service-core/src/system/ServiceContext.ts index 973e4d833..ec4269a5e 100644 --- a/packages/service-core/src/system/ServiceContext.ts +++ b/packages/service-core/src/system/ServiceContext.ts @@ -8,7 +8,7 @@ import * as routes from '../routes/routes-index.js'; import * as storage from '../storage/storage-index.js'; import { SyncContext } from '../sync/SyncContext.js'; import * as utils from '../util/util-index.js'; -import { EmitterEngine } from '../emitters/EmitterEngine.js'; +import { EventsEngine } from '../emitters/EventsEngine.js'; export interface ServiceContext { configuration: utils.ResolvedPowerSyncConfig; @@ -20,7 +20,7 @@ export interface ServiceContext { migrations: PowerSyncMigrationManager; syncContext: SyncContext; serviceMode: ServiceContextMode; - emitterEngine: EmitterEngine; + emitterEngine: EventsEngine; } export enum ServiceContextMode { @@ -47,7 +47,7 @@ export class ServiceContextContainer implements ServiceContext { configuration: utils.ResolvedPowerSyncConfig; lifeCycleEngine: LifeCycledSystem; storageEngine: storage.StorageEngine; - emitterEngine: EmitterEngine; + emitterEngine: EventsEngine; syncContext: SyncContext; routerEngine: routes.RouterEngine; serviceMode: ServiceContextMode; @@ -69,7 +69,10 @@ export class ServiceContextContainer implements ServiceContext { } }); - this.emitterEngine = new EmitterEngine(); + this.emitterEngine = new EventsEngine(); + this.lifeCycleEngine.withLifecycle(this.emitterEngine, { + stop: (emitterEngine) => emitterEngine.shutDown() + }); this.lifeCycleEngine.withLifecycle(this.storageEngine, { start: (storageEngine) => storageEngine.start(), From cbfa0cbd4ff160cca2423a550e13f56c63aed109 Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 18 Aug 2025 08:46:18 +0200 Subject: [PATCH 159/169] PR chnages --- .../src/storage/MongoReportStorage.ts | 10 ++++------ .../src/storage/implementation/models.ts | 2 +- .../test/src/sdk-report-storage.test.ts | 6 +++--- .../src/storage/PostgresReportStorageFactory.ts | 8 ++++---- .../test/src/sdk-report-storage.test.ts | 8 ++++---- .../service-core/src/routes/endpoints/socket-route.ts | 2 +- .../service-core/src/routes/endpoints/sync-stream.ts | 2 +- packages/service-core/src/storage/ReportStorage.ts | 4 ++-- packages/types/src/events.ts | 5 ++--- 9 files changed, 22 insertions(+), 25 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index c3740e16b..627f5f026 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -25,10 +25,10 @@ export class MongoReportStorage implements storage.ReportStorage { } } - async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { + async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { const { start, end } = data; const result = await this.db.sdk_report_events - .aggregate([ + .aggregate([ { $match: { connect_at: { $lte: end, $gte: start } @@ -74,12 +74,10 @@ export class MongoReportStorage implements storage.ReportStorage { } ); } - async listCurrentConnections( - data: event_types.ListCurrentConnectionsRequest - ): Promise { + async listCurrentConnections(data: event_types.ListCurrentConnectionsRequest): Promise { const timeframeFilter = this.listConnectionsDateRange(data); const result = await this.db.sdk_report_events - .aggregate([ + .aggregate([ { $match: { disconnect_at: { $exists: false }, diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index eadff6b0d..788e24496 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -221,4 +221,4 @@ export interface InstanceDocument { _id: string; } -export interface SdkConnectEventDocument extends event_types.SdkConnectDocument {} +export interface SdkConnectEventDocument extends event_types.SdkConnection {} diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts index a3dc5be21..e168299a7 100644 --- a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts @@ -3,9 +3,9 @@ import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js'; import { event_types } from '@powersync/service-types'; function removeVolatileFields( - sdks: event_types.SdkConnectDocument[] -): Partial[] { - return sdks.map((sdk: Partial) => { + sdks: event_types.SdkConnection[] +): Partial[] { + return sdks.map((sdk: Partial) => { const { _id, disconnect_at, connect_at, jwt_exp, ...rest } = sdk; return { ...rest diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 75005a838..96245fe91 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -6,10 +6,10 @@ import * as lib_postgres from '@powersync/lib-service-postgres'; import { NormalizedPostgresStorageConfig } from '../types/types.js'; import { DeleteOldSdkData, - ListCurrentConnections, ListCurrentConnectionsRequest, ScrapeSdkDataRequest, SdkConnectBucketData, + SdkConnections, SdkDisconnectEventData } from '@powersync/service-types/src/events.js'; import { SdkReporting, SdkReportingDecoded } from '../types/models/SdkReporting.js'; @@ -50,7 +50,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { }; } - private mapListCurrentConnectionsResponse(result: SdkReportingDecoded | null): ListCurrentConnections { + private mapListCurrentConnectionsResponse(result: SdkReportingDecoded | null): SdkConnections { if (!result) { return { users: 0, @@ -229,12 +229,12 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { AND connect_at = ${{ type: 1184, value: connectIsoString }} `.execute(); } - async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { + async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { const result = await this.listConnectionsQuery(data); return this.mapListCurrentConnectionsResponse(result); } - async scrapeSdkData(data: ScrapeSdkDataRequest): Promise { + async scrapeSdkData(data: ScrapeSdkDataRequest): Promise { const { start, end } = data; const result = await this.db.sql` WITH diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index 32f17bad9..82838f342 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -2,7 +2,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { POSTGRES_REPORT_STORAGE_FACTORY } from './util.js'; import { event_types } from '@powersync/service-types'; -function removeVolatileFields(sdks: event_types.SdkConnectDocument[]): Partial[] { +function removeVolatileFields(sdks: event_types.SdkConnection[]): Partial[] { return sdks.map((sdk) => { const { id, disconnect_at, connect_at, jwt_exp, ...rest } = sdk; return { @@ -268,7 +268,7 @@ describe('SDK reporting storage', async () => { }); const sdk = await factory.db - .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_one.user_id }}`.rows(); + .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_one.user_id }}`.rows(); expect(sdk).toHaveLength(1); expect(new Date(sdk[0].connect_at).toISOString()).toEqual(newConnectAt.toISOString()); expect(new Date(sdk[0].jwt_exp!).toISOString()).toEqual(jwtExp.toISOString()); @@ -297,7 +297,7 @@ describe('SDK reporting storage', async () => { }); const sdk = await factory.db - .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_three.user_id }}`.rows(); + .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_three.user_id }}`.rows(); expect(sdk).toHaveLength(1); expect(new Date(sdk[0].disconnect_at!).toISOString()).toEqual(disconnectAt.toISOString()); const cleaned = removeVolatileFields(sdk); @@ -318,7 +318,7 @@ describe('SDK reporting storage', async () => { }); const sdk = await factory.db - .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_week.user_id }}`.rows(); + .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_week.user_id }}`.rows(); expect(sdk).toHaveLength(2); const cleaned = removeVolatileFields(sdk); expect(cleaned).toMatchSnapshot(); diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 163d947ba..0be3e0b78 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -23,7 +23,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => user_agent: context.user_agent }; - const sdkData: event_types.SdkUserData = { + const sdkData: event_types.SdkUserData & event_types.SdkConnectEventData = { client_id: params.client_id, user_id: context.user_id!, user_agent: context.user_agent, diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 6978190ae..3a9ad7790 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -42,7 +42,7 @@ export const syncStreamed = routeDefinition({ user_id: payload.context.user_id, bson: useBson }; - const sdkData: event_types.SdkUserData = { + const sdkData: event_types.SdkUserData & event_types.SdkConnectEventData = { client_id: clientId, user_id: payload.context.user_id!, user_agent: userAgent as string, diff --git a/packages/service-core/src/storage/ReportStorage.ts b/packages/service-core/src/storage/ReportStorage.ts index 1b8a9ae74..a5b81a5c5 100644 --- a/packages/service-core/src/storage/ReportStorage.ts +++ b/packages/service-core/src/storage/ReportStorage.ts @@ -3,7 +3,7 @@ import { event_types } from '@powersync/service-types'; export interface ReportStorage extends AsyncDisposable { reportSdkConnect(data: event_types.SdkConnectBucketData): Promise; reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise; - listCurrentConnections(data: event_types.ListCurrentConnectionsRequest): Promise; - scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise; + listCurrentConnections(data: event_types.ListCurrentConnectionsRequest): Promise; + scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise; deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise; } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index e91bbfe8f..0baa4ca44 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -22,7 +22,6 @@ export type SdkUserData = { user_id: string; user_agent?: string; jwt_exp?: Date; - connect_at: Date; }; export type DeleteOldSdkData = { @@ -43,7 +42,7 @@ export type SdkDisconnectEventData = { connect_at: Date; } & SdkUserData; -export type SdkConnectDocument = { +export type SdkConnection = { id?: string; sdk: string; user_agent: string; @@ -54,7 +53,7 @@ export type SdkConnectDocument = { disconnect_at?: Date; }; -export type ListCurrentConnections = { +export type SdkConnections = { users: number; sdks: { sdk: string; From fa5b8720d1dcd978d4d690b237853a4ab294c99a Mon Sep 17 00:00:00 2001 From: JuanB Date: Mon, 18 Aug 2025 12:31:28 +0200 Subject: [PATCH 160/169] PR changes as requested --- .../migrations/1752661449910-sdk-reporting.ts | 4 +- .../src/storage/MongoReportStorage.ts | 23 +++---- .../test/src/sdk-report-storage.test.ts | 36 +++++------ .../migrations/scripts/1684951997326-init.ts | 6 +- .../storage/PostgresReportStorageFactory.ts | 42 ++++++------- .../test/src/sdk-report-storage.test.ts | 60 +++++++++---------- .../src/routes/endpoints/socket-route.ts | 4 +- .../src/routes/endpoints/sync-stream.ts | 6 +- packages/types/src/events.ts | 12 ++-- 9 files changed, 98 insertions(+), 95 deletions(-) diff --git a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts index 55f2d82f1..c44b4c1f7 100644 --- a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts +++ b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts @@ -13,9 +13,9 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { await db.sdk_report_events.createIndex( { - connect_at: 1, + connected_at: 1, jwt_exp: 1, - disconnect_at: 1 + disconnected_at: 1 }, { name: 'sdk_list_index' } ); diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 627f5f026..31a373012 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -15,8 +15,11 @@ export class MongoReportStorage implements storage.ReportStorage { async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise { const { date } = data; const result = await this.db.sdk_report_events.deleteMany({ - connect_at: { $lt: date }, - $or: [{ disconnect_at: { $exists: true } }, { jwt_exp: { $lt: new Date() }, disconnect_at: { $exists: false } }] + connected_at: { $lt: date }, + $or: [ + { disconnected_at: { $exists: true } }, + { jwt_exp: { $lt: new Date() }, disconnected_at: { $exists: false } } + ] }); if (result.deletedCount > 0) { logger.info( @@ -31,7 +34,7 @@ export class MongoReportStorage implements storage.ReportStorage { .aggregate([ { $match: { - connect_at: { $lte: end, $gte: start } + connected_at: { $lte: end, $gte: start } } }, this.sdkFacetPipeline(), @@ -48,7 +51,7 @@ export class MongoReportStorage implements storage.ReportStorage { { $set: data, $unset: { - disconnect_at: '' + disconnected_at: '' } }, { @@ -57,16 +60,16 @@ export class MongoReportStorage implements storage.ReportStorage { ); } async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { - const { connect_at, user_id, client_id } = data; + const { connected_at, user_id, client_id } = data; await this.db.sdk_report_events.findOneAndUpdate( { client_id, user_id, - connect_at + connected_at }, { $set: { - disconnect_at: data.disconnect_at + disconnected_at: data.disconnected_at }, $unset: { jwt_exp: '' @@ -80,7 +83,7 @@ export class MongoReportStorage implements storage.ReportStorage { .aggregate([ { $match: { - disconnect_at: { $exists: false }, + disconnected_at: { $exists: false }, jwt_exp: { $gt: new Date() }, ...timeframeFilter } @@ -165,7 +168,7 @@ export class MongoReportStorage implements storage.ReportStorage { return { user_id: userId, client_id: clientId, - connect_at: { + connected_at: { // Need to create a new date here to sett the time to 00:00:00 $gte: new Date(year, month, today), $lt: new Date(year, month, nextDay) @@ -181,7 +184,7 @@ export class MongoReportStorage implements storage.ReportStorage { const endDate = data.range?.end ? new Date(data.range.end) : new Date(); const startDate = new Date(range.start); return { - connect_at: { + connected_at: { $lte: endDate, $gte: startDate } diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts index e168299a7..7209fbdff 100644 --- a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts @@ -6,7 +6,7 @@ function removeVolatileFields( sdks: event_types.SdkConnection[] ): Partial[] { return sdks.map((sdk: Partial) => { - const { _id, disconnect_at, connect_at, jwt_exp, ...rest } = sdk; + const { _id, disconnected_at, connected_at, jwt_exp, ...rest } = sdk; return { ...rest }; @@ -37,7 +37,7 @@ describe('SDK reporting storage', async () => { const user_one = { user_id: 'user_one', client_id: 'client_one', - connect_at: now, + connected_at: now, sdk: 'powersync-dart/1.6.4', user_agent: 'powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android', jwt_exp: nowAdd5minutes @@ -45,7 +45,7 @@ describe('SDK reporting storage', async () => { const user_two = { user_id: 'user_two', client_id: 'client_two', - connect_at: nowLess5minutes, + connected_at: nowLess5minutes, sdk: 'powersync-js/1.21.1', user_agent: 'powersync-js/1.21.0 powersync-web Chromium/138 linux', jwt_exp: nowAdd5minutes @@ -53,16 +53,16 @@ describe('SDK reporting storage', async () => { const user_three = { user_id: 'user_three', client_id: 'client_three', - connect_at: yesterday, + connected_at: yesterday, sdk: 'powersync-js/1.21.2', user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', - disconnect_at: yesterday + disconnected_at: yesterday }; const user_four = { user_id: 'user_four', client_id: 'client_four', - connect_at: now, + connected_at: now, sdk: 'powersync-js/1.21.4', user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', jwt_exp: nowLess5minutes @@ -71,25 +71,25 @@ describe('SDK reporting storage', async () => { const user_week = { user_id: 'user_week', client_id: 'client_week', - connect_at: weekAgo, + connected_at: weekAgo, sdk: 'powersync-js/1.24.5', user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', - disconnect_at: weekAgo + disconnected_at: weekAgo }; const user_month = { user_id: 'user_month', client_id: 'client_month', - connect_at: monthAgo, + connected_at: monthAgo, sdk: 'powersync-js/1.23.6', user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', - disconnect_at: monthAgo + disconnected_at: monthAgo }; const user_expired = { user_id: 'user_expired', client_id: 'client_expired', - connect_at: monthAgo, + connected_at: monthAgo, sdk: 'powersync-js/1.23.7', user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', jwt_exp: monthAgo @@ -180,7 +180,7 @@ describe('SDK reporting storage', async () => { const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); await factory.reportSdkConnect({ sdk: user_one.sdk, - connect_at: newConnectAt, + connected_at: newConnectAt, jwt_exp: jwtExp, client_id: user_one.client_id, user_id: user_one.user_id, @@ -189,9 +189,9 @@ describe('SDK reporting storage', async () => { const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); expect(sdk).toHaveLength(1); - expect(new Date(sdk[0].connect_at)).toEqual(newConnectAt); + expect(new Date(sdk[0].connected_at)).toEqual(newConnectAt); expect(new Date(sdk[0].jwt_exp!)).toEqual(jwtExp); - expect(sdk[0].disconnect_at).toBeUndefined(); + expect(sdk[0].disconnected_at).toBeUndefined(); const cleaned = removeVolatileFields(sdk); expect(cleaned).toMatchSnapshot(); }); @@ -207,17 +207,17 @@ describe('SDK reporting storage', async () => { const jwtExp = new Date(disconnectAt.getFullYear(), disconnectAt.getMonth(), disconnectAt.getDate() + 1); await factory.reportSdkDisconnect({ - disconnect_at: disconnectAt, + disconnected_at: disconnectAt, jwt_exp: jwtExp, client_id: user_three.client_id, user_id: user_three.user_id, user_agent: user_three.user_agent, - connect_at: user_three.connect_at + connected_at: user_three.connected_at }); const sdk = await factory.db.sdk_report_events.find({ user_id: user_three.user_id }).toArray(); expect(sdk).toHaveLength(1); - expect(new Date(sdk[0].disconnect_at!)).toEqual(disconnectAt); + expect(new Date(sdk[0].disconnected_at!)).toEqual(disconnectAt); const cleaned = removeVolatileFields(sdk); expect(cleaned).toMatchSnapshot(); }); @@ -228,7 +228,7 @@ describe('SDK reporting storage', async () => { await factory.reportSdkConnect({ sdk: user_week.sdk, - connect_at: newConnectAt, + connected_at: newConnectAt, jwt_exp: jwtExp, client_id: user_week.client_id, user_id: user_week.user_id, diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index d54669d11..77850ba02 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -136,12 +136,12 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { user_id TEXT NOT NULL, sdk TEXT NOT NULL, jwt_exp TIMESTAMP WITH TIME ZONE, - connect_at TIMESTAMP WITH TIME ZONE NOT NULL, - disconnect_at TIMESTAMP WITH TIME ZONE + connected_at TIMESTAMP WITH TIME ZONE NOT NULL, + disconnected_at TIMESTAMP WITH TIME ZONE ) `.execute(); - await db.sql` CREATE INDEX sdk_list_index ON sdk_report_events (connect_at, jwt_exp, disconnect_at) `.execute(); + await db.sql` CREATE INDEX sdk_list_index ON sdk_report_events (connected_at, jwt_exp, disconnected_at) `.execute(); await db.sql`CREATE INDEX sdk_user_id_index ON sdk_report_events (user_id)`.execute(); diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 96245fe91..b83416522 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -73,7 +73,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { FROM sdk_report_events WHERE - disconnect_at IS NULL + disconnected_at IS NULL AND jwt_exp > NOW() ), unique_users AS ( @@ -121,10 +121,10 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { FROM sdk_report_events WHERE - disconnect_at IS NULL + disconnected_at IS NULL AND jwt_exp > NOW() - AND connect_at >= ${{ type: 1184, value: gt }} - AND connect_at <= ${{ type: 1184, value: lt }} + AND connected_at >= ${{ type: 1184, value: gt }} + AND connected_at <= ${{ type: 1184, value: lt }} ), unique_users AS ( SELECT @@ -170,24 +170,24 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { } async reportSdkConnect(data: SdkConnectBucketData): Promise { - const { sdk, connect_at, user_id, user_agent, jwt_exp, client_id } = data; - const connectIsoString = connect_at.toISOString(); + const { sdk, connected_at, user_id, user_agent, jwt_exp, client_id } = data; + const connectIsoString = connected_at.toISOString(); const jwtExpIsoString = jwt_exp!.toISOString(); const { gte, lt } = this.updateTableFilter(); const uuid = v4(); const result = await this.db.sql` UPDATE sdk_report_events SET - connect_at = ${{ type: 1184, value: connectIsoString }}, + connected_at = ${{ type: 1184, value: connectIsoString }}, sdk = ${{ type: 'varchar', value: sdk }}, user_agent = ${{ type: 'varchar', value: user_agent }}, jwt_exp = ${{ type: 1184, value: jwtExpIsoString }}, - disconnect_at = NULL + disconnected_at = NULL WHERE user_id = ${{ type: 'varchar', value: user_id }} AND client_id = ${{ type: 'varchar', value: client_id }} - AND connect_at >= ${{ type: 1184, value: gte }} - AND connect_at < ${{ type: 1184, value: lt }}; + AND connected_at >= ${{ type: 1184, value: gte }} + AND connected_at < ${{ type: 1184, value: lt }}; `.execute(); if (result.results[1].status === 'UPDATE 0') { await this.db.sql` @@ -195,7 +195,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { sdk_report_events ( user_id, client_id, - connect_at, + connected_at, sdk, user_agent, jwt_exp, @@ -215,18 +215,18 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { } } async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { - const { user_id, client_id, disconnect_at, connect_at } = data; - const disconnectIsoString = disconnect_at.toISOString(); - const connectIsoString = connect_at.toISOString(); + const { user_id, client_id, disconnected_at, connected_at } = data; + const disconnectIsoString = disconnected_at.toISOString(); + const connectIsoString = connected_at.toISOString(); await this.db.sql` UPDATE sdk_report_events SET - disconnect_at = ${{ type: 1184, value: disconnectIsoString }}, + disconnected_at = ${{ type: 1184, value: disconnectIsoString }}, jwt_exp = NULL WHERE user_id = ${{ type: 'varchar', value: user_id }} AND client_id = ${{ type: 'varchar', value: client_id }} - AND connect_at = ${{ type: 1184, value: connectIsoString }} + AND connected_at = ${{ type: 1184, value: connectIsoString }} `.execute(); } async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { @@ -244,8 +244,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { FROM sdk_report_events WHERE - connect_at >= ${{ type: 1184, value: start.toISOString() }} - AND connect_at <= ${{ type: 1184, value: end.toISOString() }} + connected_at >= ${{ type: 1184, value: start.toISOString() }} + AND connected_at <= ${{ type: 1184, value: end.toISOString() }} ), unique_users AS ( SELECT @@ -286,12 +286,12 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { const result = await this.db.sql` DELETE FROM sdk_report_events WHERE - connect_at < ${{ type: 1184, value: date.toISOString() }} + connected_at < ${{ type: 1184, value: date.toISOString() }} AND ( - disconnect_at IS NOT NULL + disconnected_at IS NOT NULL OR ( jwt_exp < NOW() - AND disconnect_at IS NULL + AND disconnected_at IS NULL ) ); `.execute(); diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index 82838f342..a6d000208 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -4,7 +4,7 @@ import { event_types } from '@powersync/service-types'; function removeVolatileFields(sdks: event_types.SdkConnection[]): Partial[] { return sdks.map((sdk) => { - const { id, disconnect_at, connect_at, jwt_exp, ...rest } = sdk; + const { id, disconnected_at, connected_at, jwt_exp, ...rest } = sdk; return { ...rest }; @@ -35,7 +35,7 @@ describe('SDK reporting storage', async () => { const user_one = { user_id: 'user_one', client_id: 'client_one', - connect_at: now.toISOString(), + connected_at: now.toISOString(), sdk: 'powersync-dart/1.6.4', user_agent: 'powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android', jwt_exp: nowAdd5minutes.toISOString(), @@ -44,7 +44,7 @@ describe('SDK reporting storage', async () => { const user_two = { user_id: 'user_two', client_id: 'client_two', - connect_at: nowLess5minutes.toISOString(), + connected_at: nowLess5minutes.toISOString(), sdk: 'powersync-js/1.21.1', user_agent: 'powersync-js/1.21.0 powersync-web Chromium/138 linux', jwt_exp: nowAdd5minutes.toISOString(), @@ -53,17 +53,17 @@ describe('SDK reporting storage', async () => { const user_three = { user_id: 'user_three', client_id: 'client_three', - connect_at: yesterday.toISOString(), + connected_at: yesterday.toISOString(), sdk: 'powersync-js/1.21.2', user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', - disconnect_at: yesterday.toISOString(), + disconnected_at: yesterday.toISOString(), id: '3' }; const user_four = { user_id: 'user_four', client_id: 'client_four', - connect_at: now.toISOString(), + connected_at: now.toISOString(), sdk: 'powersync-js/1.21.4', user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', jwt_exp: nowLess5minutes.toISOString(), @@ -73,27 +73,27 @@ describe('SDK reporting storage', async () => { const user_week = { user_id: 'user_week', client_id: 'client_week', - connect_at: weekAgo.toISOString(), + connected_at: weekAgo.toISOString(), sdk: 'powersync-js/1.24.5', user_agent: 'powersync-js/1.21.0 powersync-web Firefox/141 linux', - disconnect_at: weekAgo.toISOString(), + disconnected_at: weekAgo.toISOString(), id: 'week' }; const user_month = { user_id: 'user_month', client_id: 'client_month', - connect_at: monthAgo.toISOString(), + connected_at: monthAgo.toISOString(), sdk: 'powersync-js/1.23.6', user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', - disconnect_at: monthAgo.toISOString(), + disconnected_at: monthAgo.toISOString(), id: 'month' }; const user_expired = { user_id: 'user_expired', client_id: 'client_expired', - connect_at: monthAgo.toISOString(), + connected_at: monthAgo.toISOString(), sdk: 'powersync-js/1.23.7', user_agent: 'powersync-js/1.23.0 powersync-web Firefox/141 linux', jwt_exp: monthAgo.toISOString(), @@ -106,18 +106,18 @@ describe('SDK reporting storage', async () => { sdk_report_events ( user_id, client_id, - connect_at, + connected_at, sdk, user_agent, jwt_exp, id, - disconnect_at + disconnected_at ) VALUES ( ${{ type: 'varchar', value: user_one.user_id }}, ${{ type: 'varchar', value: user_one.client_id }}, - ${{ type: 1184, value: user_one.connect_at }}, + ${{ type: 1184, value: user_one.connected_at }}, ${{ type: 'varchar', value: user_one.sdk }}, ${{ type: 'varchar', value: user_one.user_agent }}, ${{ type: 1184, value: user_one.jwt_exp }}, @@ -127,7 +127,7 @@ describe('SDK reporting storage', async () => { ( ${{ type: 'varchar', value: user_two.user_id }}, ${{ type: 'varchar', value: user_two.client_id }}, - ${{ type: 1184, value: user_two.connect_at }}, + ${{ type: 1184, value: user_two.connected_at }}, ${{ type: 'varchar', value: user_two.sdk }}, ${{ type: 'varchar', value: user_two.user_agent }}, ${{ type: 1184, value: user_two.jwt_exp }}, @@ -137,7 +137,7 @@ describe('SDK reporting storage', async () => { ( ${{ type: 'varchar', value: user_four.user_id }}, ${{ type: 'varchar', value: user_four.client_id }}, - ${{ type: 1184, value: user_four.connect_at }}, + ${{ type: 1184, value: user_four.connected_at }}, ${{ type: 'varchar', value: user_four.sdk }}, ${{ type: 'varchar', value: user_four.user_agent }}, ${{ type: 1184, value: user_four.jwt_exp }}, @@ -147,37 +147,37 @@ describe('SDK reporting storage', async () => { ( ${{ type: 'varchar', value: user_three.user_id }}, ${{ type: 'varchar', value: user_three.client_id }}, - ${{ type: 1184, value: user_three.connect_at }}, + ${{ type: 1184, value: user_three.connected_at }}, ${{ type: 'varchar', value: user_three.sdk }}, ${{ type: 'varchar', value: user_three.user_agent }}, NULL, ${{ type: 'varchar', value: user_three.id }}, - ${{ type: 1184, value: user_three.disconnect_at }} + ${{ type: 1184, value: user_three.disconnected_at }} ), ( ${{ type: 'varchar', value: user_week.user_id }}, ${{ type: 'varchar', value: user_week.client_id }}, - ${{ type: 1184, value: user_week.connect_at }}, + ${{ type: 1184, value: user_week.connected_at }}, ${{ type: 'varchar', value: user_week.sdk }}, ${{ type: 'varchar', value: user_week.user_agent }}, NULL, ${{ type: 'varchar', value: user_week.id }}, - ${{ type: 1184, value: user_week.disconnect_at }} + ${{ type: 1184, value: user_week.disconnected_at }} ), ( ${{ type: 'varchar', value: user_month.user_id }}, ${{ type: 'varchar', value: user_month.client_id }}, - ${{ type: 1184, value: user_month.connect_at }}, + ${{ type: 1184, value: user_month.connected_at }}, ${{ type: 'varchar', value: user_month.sdk }}, ${{ type: 'varchar', value: user_month.user_agent }}, NULL, ${{ type: 'varchar', value: user_month.id }}, - ${{ type: 1184, value: user_month.disconnect_at }} + ${{ type: 1184, value: user_month.disconnected_at }} ), ( ${{ type: 'varchar', value: user_expired.user_id }}, ${{ type: 'varchar', value: user_expired.client_id }}, - ${{ type: 1184, value: user_expired.connect_at }}, + ${{ type: 1184, value: user_expired.connected_at }}, ${{ type: 'varchar', value: user_expired.sdk }}, ${{ type: 'varchar', value: user_expired.user_agent }}, ${{ type: 1184, value: user_expired.jwt_exp }}, @@ -260,7 +260,7 @@ describe('SDK reporting storage', async () => { const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); await factory.reportSdkConnect({ sdk: user_one.sdk, - connect_at: newConnectAt, + connected_at: newConnectAt, jwt_exp: jwtExp, client_id: user_one.client_id, user_id: user_one.user_id, @@ -270,9 +270,9 @@ describe('SDK reporting storage', async () => { const sdk = await factory.db .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_one.user_id }}`.rows(); expect(sdk).toHaveLength(1); - expect(new Date(sdk[0].connect_at).toISOString()).toEqual(newConnectAt.toISOString()); + expect(new Date(sdk[0].connected_at).toISOString()).toEqual(newConnectAt.toISOString()); expect(new Date(sdk[0].jwt_exp!).toISOString()).toEqual(jwtExp.toISOString()); - expect(sdk[0].disconnect_at).toBeNull(); + expect(sdk[0].disconnected_at).toBeNull(); const cleaned = removeVolatileFields(sdk); expect(cleaned).toMatchSnapshot(); }); @@ -288,18 +288,18 @@ describe('SDK reporting storage', async () => { const jwtExp = new Date(disconnectAt.getFullYear(), disconnectAt.getMonth(), disconnectAt.getDate() + 1); await factory.reportSdkDisconnect({ - disconnect_at: disconnectAt, + disconnected_at: disconnectAt, jwt_exp: jwtExp, client_id: user_three.client_id, user_id: user_three.user_id, user_agent: user_three.user_agent, - connect_at: yesterday + connected_at: yesterday }); const sdk = await factory.db .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_three.user_id }}`.rows(); expect(sdk).toHaveLength(1); - expect(new Date(sdk[0].disconnect_at!).toISOString()).toEqual(disconnectAt.toISOString()); + expect(new Date(sdk[0].disconnected_at!).toISOString()).toEqual(disconnectAt.toISOString()); const cleaned = removeVolatileFields(sdk); expect(cleaned).toMatchSnapshot(); }); @@ -310,7 +310,7 @@ describe('SDK reporting storage', async () => { await factory.reportSdkConnect({ sdk: user_week.sdk, - connect_at: newConnectAt, + connected_at: newConnectAt, jwt_exp: jwtExp, client_id: user_week.client_id, user_id: user_week.user_id, diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 0be3e0b78..49d9c7210 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -28,7 +28,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => user_id: context.user_id!, user_agent: context.user_agent, jwt_exp: context.token_payload?.exp ? new Date(context.token_payload.exp * 1000) : undefined, - connect_at: new Date(streamStart) + connected_at: new Date(streamStart) }; // Best effort guess on why the stream was closed. @@ -170,7 +170,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { ...sdkData, - disconnect_at: new Date() + disconnected_at: new Date() }); } } diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 3a9ad7790..5dd921968 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -47,7 +47,7 @@ export const syncStreamed = routeDefinition({ user_id: payload.context.user_id!, user_agent: userAgent as string, jwt_exp: token_payload?.exp ? new Date(token_payload?.exp * 1000) : undefined, - connect_at: new Date(streamStart) + connected_at: new Date(streamStart) }; if (routerEngine.closed) { @@ -140,7 +140,7 @@ export const syncStreamed = routeDefinition({ metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { ...sdkData, - disconnect_at: new Date() + disconnected_at: new Date() }); logger.info(`Sync stream complete`, { ...tracker.getLogMeta(), @@ -154,7 +154,7 @@ export const syncStreamed = routeDefinition({ metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { ...sdkData, - disconnect_at: new Date() + disconnected_at: new Date() }); } } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts index 0baa4ca44..c21b72201 100644 --- a/packages/types/src/events.ts +++ b/packages/types/src/events.ts @@ -29,17 +29,17 @@ export type DeleteOldSdkData = { }; export type SdkConnectEventData = { - connect_at: Date; + connected_at: Date; } & SdkUserData; export type SdkConnectBucketData = { - connect_at: Date; + connected_at: Date; sdk: string; } & SdkUserData; export type SdkDisconnectEventData = { - disconnect_at: Date; - connect_at: Date; + disconnected_at: Date; + connected_at: Date; } & SdkUserData; export type SdkConnection = { @@ -49,8 +49,8 @@ export type SdkConnection = { client_id: string; user_id: string; jwt_exp?: Date; - connect_at: Date; - disconnect_at?: Date; + connected_at: Date; + disconnected_at?: Date; }; export type SdkConnections = { From a15ee2da6a63b46614d203d3cb7119b9849ae52b Mon Sep 17 00:00:00 2001 From: JuanB Date: Tue, 19 Aug 2025 15:19:25 +0200 Subject: [PATCH 161/169] changed names to the agreed naming convention --- .../src/storage/MongoReportStorage.ts | 30 ++++---- .../src/storage/implementation/db.ts | 4 +- .../src/storage/implementation/models.ts | 2 +- .../test/src/sdk-report-storage.test.ts | 26 +++---- modules/module-postgres-storage/package.json | 2 +- .../storage/PostgresReportStorageFactory.ts | 24 +++--- .../test/src/sdk-report-storage.test.ts | 28 +++---- package.json | 1 + .../src/{emitters => events}/EventsEngine.ts | 8 +- .../src/routes/endpoints/socket-route.ts | 7 +- .../src/routes/endpoints/sync-stream.ts | 9 +-- .../service-core/src/storage/ReportStorage.ts | 12 +-- .../src/storage/StorageProvider.ts | 3 +- .../service-core/src/system/ServiceContext.ts | 12 +-- packages/types/src/events.ts | 75 ------------------- packages/types/src/index.ts | 2 +- packages/types/src/reports.ts | 75 +++++++++++++++++++ 17 files changed, 158 insertions(+), 162 deletions(-) rename packages/service-core/src/{emitters => events}/EventsEngine.ts (76%) delete mode 100644 packages/types/src/events.ts create mode 100644 packages/types/src/reports.ts diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 31a373012..5168b32fe 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -12,7 +12,7 @@ export class MongoReportStorage implements storage.ReportStorage { this.client = db.client; this.db = db; } - async deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise { + async deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise { const { date } = data; const result = await this.db.sdk_report_events.deleteMany({ connected_at: { $lt: date }, @@ -28,23 +28,25 @@ export class MongoReportStorage implements storage.ReportStorage { } } - async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise { + async getClientConnectionReports( + data: event_types.ClientConnectionReportRequest + ): Promise { const { start, end } = data; const result = await this.db.sdk_report_events - .aggregate([ + .aggregate([ { $match: { connected_at: { $lte: end, $gte: start } } }, - this.sdkFacetPipeline(), - this.sdkProjectPipeline() + this.connectionsFacetPipeline(), + this.connectionsProjectPipeline() ]) .toArray(); return result[0]; } - async reportSdkConnect(data: event_types.SdkConnectBucketData): Promise { + async reportClientConnection(data: event_types.ClientConnectionBucketData): Promise { const updateFilter = this.updateDocFilter(data.user_id, data.client_id!); await this.db.sdk_report_events.findOneAndUpdate( updateFilter, @@ -59,7 +61,7 @@ export class MongoReportStorage implements storage.ReportStorage { } ); } - async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise { + async reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise { const { connected_at, user_id, client_id } = data; await this.db.sdk_report_events.findOneAndUpdate( { @@ -77,10 +79,10 @@ export class MongoReportStorage implements storage.ReportStorage { } ); } - async listCurrentConnections(data: event_types.ListCurrentConnectionsRequest): Promise { + async getConnectedClients(data: event_types.ClientConnectionsRequest): Promise { const timeframeFilter = this.listConnectionsDateRange(data); const result = await this.db.sdk_report_events - .aggregate([ + .aggregate([ { $match: { disconnected_at: { $exists: false }, @@ -88,8 +90,8 @@ export class MongoReportStorage implements storage.ReportStorage { ...timeframeFilter } }, - this.sdkFacetPipeline(), - this.sdkProjectPipeline() + this.connectionsFacetPipeline(), + this.connectionsProjectPipeline() ]) .toArray(); return result[0]; @@ -113,7 +115,7 @@ export class MongoReportStorage implements storage.ReportStorage { }; } - private sdkFacetPipeline() { + private connectionsFacetPipeline() { return { $facet: { unique_users: [ @@ -153,7 +155,7 @@ export class MongoReportStorage implements storage.ReportStorage { }; } - private sdkProjectPipeline() { + private connectionsProjectPipeline() { return { $project: { users: { $ifNull: [{ $arrayElemAt: ['$unique_users.count', 0] }, 0] }, @@ -176,7 +178,7 @@ export class MongoReportStorage implements storage.ReportStorage { }; } - private listConnectionsDateRange(data: event_types.ListCurrentConnectionsRequest) { + private listConnectionsDateRange(data: event_types.ClientConnectionsRequest) { const { range } = data; if (!range) { return undefined; diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 275c4fd8a..49c3eaa9a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -8,11 +8,11 @@ import { BucketParameterDocument, BucketStateDocument, CheckpointEventDocument, + ClientConnectionDocument, CurrentDataDocument, CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, - SdkConnectEventDocument, SourceTableDocument, SyncRuleDocument, WriteCheckpointDocument @@ -38,7 +38,7 @@ export class PowerSyncMongo { readonly locks: mongo.Collection; readonly bucket_state: mongo.Collection; readonly checkpoint_events: mongo.Collection; - readonly sdk_report_events: mongo.Collection; + readonly sdk_report_events: mongo.Collection; readonly client: mongo.MongoClient; readonly db: mongo.Db; diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 788e24496..46da8adb5 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -221,4 +221,4 @@ export interface InstanceDocument { _id: string; } -export interface SdkConnectEventDocument extends event_types.SdkConnection {} +export interface ClientConnectionDocument extends event_types.ClientConnection {} diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts index 7209fbdff..68cfd17ff 100644 --- a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts @@ -3,9 +3,9 @@ import { INITIALIZED_MONGO_REPORT_STORAGE_FACTORY } from './util.js'; import { event_types } from '@powersync/service-types'; function removeVolatileFields( - sdks: event_types.SdkConnection[] -): Partial[] { - return sdks.map((sdk: Partial) => { + sdks: event_types.ClientConnection[] +): Partial[] { + return sdks.map((sdk: Partial) => { const { _id, disconnected_at, connected_at, jwt_exp, ...rest } = sdk; return { ...rest @@ -119,7 +119,7 @@ describe('SDK reporting storage', async () => { await deleteData(); }); it('Should show connected users with start range', async () => { - const current = await factory.listCurrentConnections({ + const current = await factory.getConnectedClients({ range: { start: new Date( now.getFullYear(), @@ -133,7 +133,7 @@ describe('SDK reporting storage', async () => { expect(current).toMatchSnapshot(); }); it('Should show connected users with start range and end range', async () => { - const current = await factory.listCurrentConnections({ + const current = await factory.getConnectedClients({ range: { end: nowLess5minutes.toISOString(), start: new Date( @@ -148,21 +148,21 @@ describe('SDK reporting storage', async () => { expect(current).toMatchSnapshot(); }); it('Should show SDK scrape data for user over the past month', async () => { - const sdk = await factory.scrapeSdkData({ + const sdk = await factory.getClientConnectionReports({ start: monthAgo, end: now }); expect(sdk).toMatchSnapshot(); }); it('Should show SDK scrape data for user over the past week', async () => { - const sdk = await factory.scrapeSdkData({ + const sdk = await factory.getClientConnectionReports({ start: weekAgo, end: now }); expect(sdk).toMatchSnapshot(); }); it('Should show SDK scrape data for user over the past day', async () => { - const sdk = await factory.scrapeSdkData({ + const sdk = await factory.getClientConnectionReports({ start: dayAgo, end: now }); @@ -178,7 +178,7 @@ describe('SDK reporting storage', async () => { now.getMinutes() + 20 ); const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); - await factory.reportSdkConnect({ + await factory.reportClientConnection({ sdk: user_one.sdk, connected_at: newConnectAt, jwt_exp: jwtExp, @@ -206,7 +206,7 @@ describe('SDK reporting storage', async () => { ); const jwtExp = new Date(disconnectAt.getFullYear(), disconnectAt.getMonth(), disconnectAt.getDate() + 1); - await factory.reportSdkDisconnect({ + await factory.reportClientDisconnection({ disconnected_at: disconnectAt, jwt_exp: jwtExp, client_id: user_three.client_id, @@ -226,7 +226,7 @@ describe('SDK reporting storage', async () => { const newConnectAt = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, now.getHours()); const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); - await factory.reportSdkConnect({ + await factory.reportClientConnection({ sdk: user_week.sdk, connected_at: newConnectAt, jwt_exp: jwtExp, @@ -244,10 +244,10 @@ describe('SDK reporting storage', async () => { it('Should delete rows older than specified range', async () => { await deleteData(); await loadData(); - await factory.deleteOldSdkData({ + await factory.deleteOldConnectionData({ date: weekAgo }); - const sdk = await factory.scrapeSdkData({ + const sdk = await factory.getClientConnectionReports({ start: monthAgo, end: now }); diff --git a/modules/module-postgres-storage/package.json b/modules/module-postgres-storage/package.json index c66988767..4b1b37adf 100644 --- a/modules/module-postgres-storage/package.json +++ b/modules/module-postgres-storage/package.json @@ -11,7 +11,7 @@ "scripts": { "build": "tsc -b", "build:tests": "tsc -b test/tsconfig.json", - "clean": "rm -rf ./lib && tsc -b --clean", + "clean": "rm -rf ./dist && tsc -b --clean", "test": "vitest" }, "exports": { diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index b83416522..683528c83 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -4,14 +4,6 @@ import { event_types } from '@powersync/service-types'; import { v4 } from 'uuid'; import * as lib_postgres from '@powersync/lib-service-postgres'; import { NormalizedPostgresStorageConfig } from '../types/types.js'; -import { - DeleteOldSdkData, - ListCurrentConnectionsRequest, - ScrapeSdkDataRequest, - SdkConnectBucketData, - SdkConnections, - SdkDisconnectEventData -} from '@powersync/service-types/src/events.js'; import { SdkReporting, SdkReportingDecoded } from '../types/models/SdkReporting.js'; import { toInteger } from 'ix/util/tointeger.js'; import { logger } from '@powersync/lib-services-framework'; @@ -50,7 +42,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { }; } - private mapListCurrentConnectionsResponse(result: SdkReportingDecoded | null): SdkConnections { + private mapListCurrentConnectionsResponse(result: SdkReportingDecoded | null): event_types.ClientConnectionReport { if (!result) { return { users: 0, @@ -62,7 +54,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { sdks: result.sdks?.data || [] }; } - private async listConnectionsQuery(data: event_types.ListCurrentConnectionsRequest) { + private async listConnectionsQuery(data: event_types.ClientConnectionsRequest) { const { range } = data; if (!range) { return this.db.sql` @@ -169,7 +161,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { }; } - async reportSdkConnect(data: SdkConnectBucketData): Promise { + async reportClientConnection(data: event_types.ClientConnectionBucketData): Promise { const { sdk, connected_at, user_id, user_agent, jwt_exp, client_id } = data; const connectIsoString = connected_at.toISOString(); const jwtExpIsoString = jwt_exp!.toISOString(); @@ -214,7 +206,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { `.execute(); } } - async reportSdkDisconnect(data: SdkDisconnectEventData): Promise { + async reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise { const { user_id, client_id, disconnected_at, connected_at } = data; const disconnectIsoString = disconnected_at.toISOString(); const connectIsoString = connected_at.toISOString(); @@ -229,12 +221,14 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { AND connected_at = ${{ type: 1184, value: connectIsoString }} `.execute(); } - async listCurrentConnections(data: ListCurrentConnectionsRequest): Promise { + async getConnectedClients(data: event_types.ClientConnectionsRequest): Promise { const result = await this.listConnectionsQuery(data); return this.mapListCurrentConnectionsResponse(result); } - async scrapeSdkData(data: ScrapeSdkDataRequest): Promise { + async getClientConnectionReports( + data: event_types.ClientConnectionReportRequest + ): Promise { const { start, end } = data; const result = await this.db.sql` WITH @@ -281,7 +275,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { .first(); return this.mapListCurrentConnectionsResponse(result); } - async deleteOldSdkData(data: DeleteOldSdkData): Promise { + async deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise { const { date } = data; const result = await this.db.sql` DELETE FROM sdk_report_events diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts index a6d000208..38efdd37e 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts @@ -2,7 +2,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { POSTGRES_REPORT_STORAGE_FACTORY } from './util.js'; import { event_types } from '@powersync/service-types'; -function removeVolatileFields(sdks: event_types.SdkConnection[]): Partial[] { +function removeVolatileFields(sdks: event_types.ClientConnection[]): Partial[] { return sdks.map((sdk) => { const { id, disconnected_at, connected_at, jwt_exp, ...rest } = sdk; return { @@ -199,7 +199,7 @@ describe('SDK reporting storage', async () => { await deleteData(); }); it('Should show connected users with start range', async () => { - const current = await factory.listCurrentConnections({ + const current = await factory.getConnectedClients({ range: { start: new Date( now.getFullYear(), @@ -213,7 +213,7 @@ describe('SDK reporting storage', async () => { expect(current).toMatchSnapshot(); }); it('Should show connected users with start range and end range', async () => { - const current = await factory.listCurrentConnections({ + const current = await factory.getConnectedClients({ range: { end: nowLess5minutes.toISOString(), start: new Date( @@ -228,21 +228,21 @@ describe('SDK reporting storage', async () => { expect(current).toMatchSnapshot(); }); it('Should show SDK scrape data for user over the past month', async () => { - const sdk = await factory.scrapeSdkData({ + const sdk = await factory.getClientConnectionReports({ start: monthAgo, end: now }); expect(sdk).toMatchSnapshot(); }); it('Should show SDK scrape data for user over the past week', async () => { - const sdk = await factory.scrapeSdkData({ + const sdk = await factory.getClientConnectionReports({ start: weekAgo, end: now }); expect(sdk).toMatchSnapshot(); }); it('Should show SDK scrape data for user over the past day', async () => { - const sdk = await factory.scrapeSdkData({ + const sdk = await factory.getClientConnectionReports({ start: dayAgo, end: now }); @@ -258,7 +258,7 @@ describe('SDK reporting storage', async () => { now.getMinutes() + 20 ); const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); - await factory.reportSdkConnect({ + await factory.reportClientConnection({ sdk: user_one.sdk, connected_at: newConnectAt, jwt_exp: jwtExp, @@ -268,7 +268,7 @@ describe('SDK reporting storage', async () => { }); const sdk = await factory.db - .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_one.user_id }}`.rows(); + .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_one.user_id }}`.rows(); expect(sdk).toHaveLength(1); expect(new Date(sdk[0].connected_at).toISOString()).toEqual(newConnectAt.toISOString()); expect(new Date(sdk[0].jwt_exp!).toISOString()).toEqual(jwtExp.toISOString()); @@ -287,7 +287,7 @@ describe('SDK reporting storage', async () => { ); const jwtExp = new Date(disconnectAt.getFullYear(), disconnectAt.getMonth(), disconnectAt.getDate() + 1); - await factory.reportSdkDisconnect({ + await factory.reportClientDisconnection({ disconnected_at: disconnectAt, jwt_exp: jwtExp, client_id: user_three.client_id, @@ -297,7 +297,7 @@ describe('SDK reporting storage', async () => { }); const sdk = await factory.db - .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_three.user_id }}`.rows(); + .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_three.user_id }}`.rows(); expect(sdk).toHaveLength(1); expect(new Date(sdk[0].disconnected_at!).toISOString()).toEqual(disconnectAt.toISOString()); const cleaned = removeVolatileFields(sdk); @@ -308,7 +308,7 @@ describe('SDK reporting storage', async () => { const newConnectAt = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, now.getHours()); const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); - await factory.reportSdkConnect({ + await factory.reportClientConnection({ sdk: user_week.sdk, connected_at: newConnectAt, jwt_exp: jwtExp, @@ -318,7 +318,7 @@ describe('SDK reporting storage', async () => { }); const sdk = await factory.db - .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_week.user_id }}`.rows(); + .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_week.user_id }}`.rows(); expect(sdk).toHaveLength(2); const cleaned = removeVolatileFields(sdk); expect(cleaned).toMatchSnapshot(); @@ -327,10 +327,10 @@ describe('SDK reporting storage', async () => { it('Should delete rows older than specified range', async () => { await deleteData(); await loadData(); - await factory.deleteOldSdkData({ + await factory.deleteOldConnectionData({ date: weekAgo }); - const sdk = await factory.scrapeSdkData({ + const sdk = await factory.getClientConnectionReports({ start: monthAgo, end: now }); diff --git a/package.json b/package.json index 1d151f566..1d379cdc8 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "watch:ts": "pnpm build:ts -w --preserveWatchOutput", "watch:service": "concurrently --passthrough-arguments \"pnpm watch:ts\" \" pnpm start:service {@}\" -- ", "start:service": "pnpm --filter @powersync/service-image watch", + "clean:modules": "rm -rf node_modules && pnpm -r exec rm -rf node_modules", "clean": "pnpm run -r clean", "release": "pnpm build:production && pnpm changeset publish", "test": "pnpm run -r test", diff --git a/packages/service-core/src/emitters/EventsEngine.ts b/packages/service-core/src/events/EventsEngine.ts similarity index 76% rename from packages/service-core/src/emitters/EventsEngine.ts rename to packages/service-core/src/events/EventsEngine.ts index 22b747de9..f67dbcb12 100644 --- a/packages/service-core/src/emitters/EventsEngine.ts +++ b/packages/service-core/src/events/EventsEngine.ts @@ -4,7 +4,7 @@ import { event_types } from '@powersync/service-types'; export class EventsEngine { private emitter: EventEmitter; - private events: Set = new Set(); + private events: Set = new Set(); constructor() { this.emitter = new EventEmitter({ captureRejections: true }); this.emitter.on('error', (error: Error) => { @@ -12,18 +12,18 @@ export class EventsEngine { }); } - subscribe(event: event_types.EmitterEvent): void { + subscribe(event: event_types.EmitterEvent): void { if (!this.events.has(event.event)) { this.events.add(event.event); } this.emitter.on(event.event, event.handler.bind(event)); } - get listEvents(): event_types.EmitterEngineEvents[] { + get listEvents(): event_types.EventsEngineEventType[] { return Array.from(this.events.values()); } - countListeners(eventName: event_types.EmitterEngineEvents): number { + countListeners(eventName: event_types.EventsEngineEventType): number { return this.emitter.listenerCount(eventName); } diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index abd30407e..4b0e3ed59 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -1,5 +1,4 @@ import { ErrorCode, errors, schema } from '@powersync/lib-services-framework'; -import { RequestParameters } from '@powersync/service-sync-rules'; import * as sync from '../../sync/sync-index.js'; import * as util from '../../util/util-index.js'; @@ -23,7 +22,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => user_agent: context.user_agent }; - const sdkData: event_types.SdkUserData & event_types.SdkConnectEventData = { + const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = { client_id: params.client_id, user_id: context.user_id!, user_agent: context.user_agent, @@ -91,7 +90,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => }); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, sdkData); + service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_CONNECT_EVENT, sdkData); const tracker = new sync.RequestTracker(metricsEngine); try { for await (const data of sync.streamResponse({ @@ -165,7 +164,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => close_reason: closeReason ?? 'unknown' }); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { + service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, { ...sdkData, disconnected_at: new Date() }); diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 2a0143b43..5ff5f78eb 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -1,5 +1,4 @@ import { ErrorCode, errors, router, schema } from '@powersync/lib-services-framework'; -import { RequestParameters } from '@powersync/service-sync-rules'; import { Readable } from 'stream'; import Negotiator from 'negotiator'; @@ -42,7 +41,7 @@ export const syncStreamed = routeDefinition({ user_id: payload.context.user_id, bson: useBson }; - const sdkData: event_types.SdkUserData & event_types.SdkConnectEventData = { + const sdkData: event_types.ConnectedUserData & event_types.ClientConnectionEventData = { client_id: clientId, user_id: payload.context.user_id!, user_agent: userAgent as string, @@ -74,7 +73,7 @@ export const syncStreamed = routeDefinition({ const tracker = new sync.RequestTracker(metricsEngine); try { metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_CONNECT_EVENT, sdkData); + service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_CONNECT_EVENT, sdkData); const syncLines = sync.streamResponse({ syncContext: syncContext, bucketStorage, @@ -134,7 +133,7 @@ export const syncStreamed = routeDefinition({ } controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { + service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, { ...sdkData, disconnected_at: new Date() }); @@ -148,7 +147,7 @@ export const syncStreamed = routeDefinition({ } catch (ex) { controller.abort(); metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); - service_context.emitterEngine.emit(event_types.EmitterEngineEvents.SDK_DISCONNECT_EVENT, { + service_context.eventsEngine.emit(event_types.EventsEngineEventType.SDK_DISCONNECT_EVENT, { ...sdkData, disconnected_at: new Date() }); diff --git a/packages/service-core/src/storage/ReportStorage.ts b/packages/service-core/src/storage/ReportStorage.ts index a5b81a5c5..df6d5838c 100644 --- a/packages/service-core/src/storage/ReportStorage.ts +++ b/packages/service-core/src/storage/ReportStorage.ts @@ -1,9 +1,11 @@ import { event_types } from '@powersync/service-types'; export interface ReportStorage extends AsyncDisposable { - reportSdkConnect(data: event_types.SdkConnectBucketData): Promise; - reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise; - listCurrentConnections(data: event_types.ListCurrentConnectionsRequest): Promise; - scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise; - deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise; + reportClientConnection(data: event_types.ClientConnectionBucketData): Promise; + reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise; + getConnectedClients(data: event_types.ClientConnectionsRequest): Promise; + getClientConnectionReports( + data: event_types.ClientConnectionReportRequest + ): Promise; + deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise; } diff --git a/packages/service-core/src/storage/StorageProvider.ts b/packages/service-core/src/storage/StorageProvider.ts index 6a32e45d2..f2404bdf0 100644 --- a/packages/service-core/src/storage/StorageProvider.ts +++ b/packages/service-core/src/storage/StorageProvider.ts @@ -5,8 +5,7 @@ import { ReportStorage } from './ReportStorage.js'; export interface ActiveStorage { storage: BucketStorageFactory; - // TODO: REMOVE THE NULL ONCE POSTGRES HAS BEEN IMPLEMENTED THIS IS JUST SO I CAN TEST MONGO - reportStorage: ReportStorage | null; + reportStorage: ReportStorage; shutDown(): Promise; /** diff --git a/packages/service-core/src/system/ServiceContext.ts b/packages/service-core/src/system/ServiceContext.ts index ec4269a5e..763fcbca0 100644 --- a/packages/service-core/src/system/ServiceContext.ts +++ b/packages/service-core/src/system/ServiceContext.ts @@ -8,7 +8,7 @@ import * as routes from '../routes/routes-index.js'; import * as storage from '../storage/storage-index.js'; import { SyncContext } from '../sync/SyncContext.js'; import * as utils from '../util/util-index.js'; -import { EventsEngine } from '../emitters/EventsEngine.js'; +import { EventsEngine } from '../events/EventsEngine.js'; export interface ServiceContext { configuration: utils.ResolvedPowerSyncConfig; @@ -20,7 +20,7 @@ export interface ServiceContext { migrations: PowerSyncMigrationManager; syncContext: SyncContext; serviceMode: ServiceContextMode; - emitterEngine: EventsEngine; + eventsEngine: EventsEngine; } export enum ServiceContextMode { @@ -47,7 +47,7 @@ export class ServiceContextContainer implements ServiceContext { configuration: utils.ResolvedPowerSyncConfig; lifeCycleEngine: LifeCycledSystem; storageEngine: storage.StorageEngine; - emitterEngine: EventsEngine; + eventsEngine: EventsEngine; syncContext: SyncContext; routerEngine: routes.RouterEngine; serviceMode: ServiceContextMode; @@ -69,8 +69,8 @@ export class ServiceContextContainer implements ServiceContext { } }); - this.emitterEngine = new EventsEngine(); - this.lifeCycleEngine.withLifecycle(this.emitterEngine, { + this.eventsEngine = new EventsEngine(); + this.lifeCycleEngine.withLifecycle(this.eventsEngine, { stop: (emitterEngine) => emitterEngine.shutDown() }); @@ -98,7 +98,7 @@ export class ServiceContextContainer implements ServiceContext { start: () => migrationManager[Symbol.asyncDispose]() }); - this.lifeCycleEngine.withLifecycle(this.emitterEngine, { + this.lifeCycleEngine.withLifecycle(this.eventsEngine, { stop: (emitterEngine) => emitterEngine.shutDown() }); } diff --git a/packages/types/src/events.ts b/packages/types/src/events.ts deleted file mode 100644 index c21b72201..000000000 --- a/packages/types/src/events.ts +++ /dev/null @@ -1,75 +0,0 @@ -export enum EmitterEngineEvents { - SDK_CONNECT_EVENT = 'sdk-connect-event', - SDK_DISCONNECT_EVENT = 'sdk-disconnect-event', - SDK_DELETE_OLD = 'sdk-delete-old' -} - -export type TimeFrames = 'hour' | 'day' | 'week' | 'month'; -export type SubscribeEvents = { - [EmitterEngineEvents.SDK_CONNECT_EVENT]: SdkConnectEventData; - [EmitterEngineEvents.SDK_DISCONNECT_EVENT]: SdkDisconnectEventData; - [EmitterEngineEvents.SDK_DELETE_OLD]: DeleteOldSdkData; -}; - -export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; -export interface EmitterEvent { - event: K; - handler: EventHandlerFunc; -} - -export type SdkUserData = { - client_id?: string; - user_id: string; - user_agent?: string; - jwt_exp?: Date; -}; - -export type DeleteOldSdkData = { - date: Date; -}; - -export type SdkConnectEventData = { - connected_at: Date; -} & SdkUserData; - -export type SdkConnectBucketData = { - connected_at: Date; - sdk: string; -} & SdkUserData; - -export type SdkDisconnectEventData = { - disconnected_at: Date; - connected_at: Date; -} & SdkUserData; - -export type SdkConnection = { - id?: string; - sdk: string; - user_agent: string; - client_id: string; - user_id: string; - jwt_exp?: Date; - connected_at: Date; - disconnected_at?: Date; -}; - -export type SdkConnections = { - users: number; - sdks: { - sdk: string; - users: number; - clients: number; - }[]; -}; - -export type ScrapeSdkDataRequest = { - start: Date; - end: Date; -}; - -export type ListCurrentConnectionsRequest = { - range?: { - start: string; - end?: string; - }; -}; diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 1f600c259..fb72c079f 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -4,4 +4,4 @@ export * from './definitions.js'; export * as internal_routes from './routes.js'; export * from './metrics.js'; export * as metric_types from './metrics.js'; -export * as event_types from './events.js'; +export * as event_types from './reports'; diff --git a/packages/types/src/reports.ts b/packages/types/src/reports.ts new file mode 100644 index 000000000..51df86316 --- /dev/null +++ b/packages/types/src/reports.ts @@ -0,0 +1,75 @@ +export enum EventsEngineEventType { + SDK_CONNECT_EVENT = 'sdk-connect-event', + SDK_DISCONNECT_EVENT = 'sdk-disconnect-event', + SDK_DELETE_OLD = 'sdk-delete-old' +} + +export type TimeFrames = 'hour' | 'day' | 'week' | 'month'; +export type SubscribeEvents = { + [EventsEngineEventType.SDK_CONNECT_EVENT]: ClientConnectionEventData; + [EventsEngineEventType.SDK_DISCONNECT_EVENT]: ClientDisconnectionEventData; + [EventsEngineEventType.SDK_DELETE_OLD]: DeleteOldConnectionData; +}; + +export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; +export interface EmitterEvent { + event: K; + handler: EventHandlerFunc; +} + +export type ConnectedUserData = { + client_id?: string; + user_id: string; + user_agent?: string; + jwt_exp?: Date; +}; + +export type DeleteOldConnectionData = { + date: Date; +}; + +export type ClientConnectionEventData = { + connected_at: Date; +} & ConnectedUserData; + +export type ClientConnectionBucketData = { + connected_at: Date; + sdk: string; +} & ConnectedUserData; + +export type ClientDisconnectionEventData = { + disconnected_at: Date; + connected_at: Date; +} & ConnectedUserData; + +export type ClientConnection = { + id?: string; + sdk: string; + user_agent: string; + client_id: string; + user_id: string; + jwt_exp?: Date; + connected_at: Date; + disconnected_at?: Date; +}; + +export type ClientConnectionReport = { + users: number; + sdks: { + sdk: string; + users: number; + clients: number; + }[]; +}; + +export type ClientConnectionReportRequest = { + start: Date; + end: Date; +}; + +export type ClientConnectionsRequest = { + range?: { + start: string; + end?: string; + }; +}; From ed65508a5e5becadf2ae10dccfbaf49b727c6ff5 Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 20 Aug 2025 12:29:57 +0200 Subject: [PATCH 162/169] chnages table names to match naming conventions --- .../1752661449910-connection-reporting.ts | 70 +++++++ .../migrations/1752661449910-sdk-reporting.ts | 70 ------- .../src/storage/MongoReportStorage.ts | 12 +- .../MongoTestReportStorageFactoryGenerator.ts | 2 +- .../src/storage/implementation/db.ts | 12 +- .../connection-report-storage.test.ts.snap | 198 ++++++++++++++++++ ...t.ts => connection-report-storage.test.ts} | 22 +- .../migrations/scripts/1684951997326-init.ts | 12 +- .../storage/PostgresReportStorageFactory.ts | 16 +- .../module-postgres-storage/src/utils/db.ts | 2 +- .../connection-report-storage.test.ts.snap | 198 ++++++++++++++++++ ...t.ts => connection-report-storage.test.ts} | 24 +-- 12 files changed, 518 insertions(+), 120 deletions(-) create mode 100644 modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-connection-reporting.ts delete mode 100644 modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts create mode 100644 modules/module-mongodb-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap rename modules/module-mongodb-storage/test/src/{sdk-report-storage.test.ts => connection-report-storage.test.ts} (88%) create mode 100644 modules/module-postgres-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap rename modules/module-postgres-storage/test/src/{sdk-report-storage.test.ts => connection-report-storage.test.ts} (90%) diff --git a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-connection-reporting.ts b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-connection-reporting.ts new file mode 100644 index 000000000..6459bc7ef --- /dev/null +++ b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-connection-reporting.ts @@ -0,0 +1,70 @@ +import { migrations } from '@powersync/service-core'; +import * as storage from '../../../storage/storage-index.js'; +import { MongoStorageConfig } from '../../../types/types.js'; + +export const up: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig); + + try { + await db.createConnectionReportingCollection(); + + await db.connection_report_events.createIndex( + { + connected_at: 1, + jwt_exp: 1, + disconnected_at: 1 + }, + { name: 'connection_list_index' } + ); + + await db.connection_report_events.createIndex( + { + user_id: 1 + }, + { name: 'connection_user_id_index' } + ); + await db.connection_report_events.createIndex( + { + client_id: 1 + }, + { name: 'connection_client_id_index' } + ); + await db.connection_report_events.createIndex( + { + sdk: 1 + }, + { name: 'connection_index' } + ); + } finally { + await db.client.close(); + } +}; + +export const down: migrations.PowerSyncMigrationFunction = async (context) => { + const { + service_context: { configuration } + } = context; + + const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig); + + try { + if (await db.connection_report_events.indexExists('connection_list_index')) { + await db.connection_report_events.dropIndex('connection_list_index'); + } + if (await db.connection_report_events.indexExists('connection_user_id_index')) { + await db.connection_report_events.dropIndex('connection_user_id_index'); + } + if (await db.connection_report_events.indexExists('connection_client_id_index')) { + await db.connection_report_events.dropIndex('connection_client_id_index'); + } + if (await db.connection_report_events.indexExists('connection_index')) { + await db.connection_report_events.dropIndex('connection_index'); + } + await db.db.dropCollection('connection_report_events'); + } finally { + await db.client.close(); + } +}; diff --git a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts deleted file mode 100644 index c44b4c1f7..000000000 --- a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-sdk-reporting.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { migrations } from '@powersync/service-core'; -import * as storage from '../../../storage/storage-index.js'; -import { MongoStorageConfig } from '../../../types/types.js'; - -export const up: migrations.PowerSyncMigrationFunction = async (context) => { - const { - service_context: { configuration } - } = context; - const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig); - - try { - await db.createSdkReportingCollection(); - - await db.sdk_report_events.createIndex( - { - connected_at: 1, - jwt_exp: 1, - disconnected_at: 1 - }, - { name: 'sdk_list_index' } - ); - - await db.sdk_report_events.createIndex( - { - user_id: 1 - }, - { name: 'sdk_user_id_index' } - ); - await db.sdk_report_events.createIndex( - { - client_id: 1 - }, - { name: 'sdk_client_id_index' } - ); - await db.sdk_report_events.createIndex( - { - sdk: 1 - }, - { name: 'sdk_index' } - ); - } finally { - await db.client.close(); - } -}; - -export const down: migrations.PowerSyncMigrationFunction = async (context) => { - const { - service_context: { configuration } - } = context; - - const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig); - - try { - if (await db.sdk_report_events.indexExists('sdk_list_index')) { - await db.sdk_report_events.dropIndex('sdk_list_index'); - } - if (await db.sdk_report_events.indexExists('sdk_user_id_index')) { - await db.sdk_report_events.dropIndex('sdk_user_id_index'); - } - if (await db.sdk_report_events.indexExists('sdk_client_id_index')) { - await db.sdk_report_events.dropIndex('sdk_client_id_index'); - } - if (await db.sdk_report_events.indexExists('sdk_index')) { - await db.sdk_report_events.dropIndex('sdk_index'); - } - await db.db.dropCollection('sdk_report_events'); - } finally { - await db.client.close(); - } -}; diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 5168b32fe..02c2aac6b 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -14,7 +14,7 @@ export class MongoReportStorage implements storage.ReportStorage { } async deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise { const { date } = data; - const result = await this.db.sdk_report_events.deleteMany({ + const result = await this.db.connection_report_events.deleteMany({ connected_at: { $lt: date }, $or: [ { disconnected_at: { $exists: true } }, @@ -23,7 +23,7 @@ export class MongoReportStorage implements storage.ReportStorage { }); if (result.deletedCount > 0) { logger.info( - `TTL from ${date.toISOString()}: ${result.deletedCount} MongoDB documents have been removed from sdk_report_events.` + `TTL from ${date.toISOString()}: ${result.deletedCount} MongoDB documents have been removed from connection_report_events.` ); } } @@ -32,7 +32,7 @@ export class MongoReportStorage implements storage.ReportStorage { data: event_types.ClientConnectionReportRequest ): Promise { const { start, end } = data; - const result = await this.db.sdk_report_events + const result = await this.db.connection_report_events .aggregate([ { $match: { @@ -48,7 +48,7 @@ export class MongoReportStorage implements storage.ReportStorage { async reportClientConnection(data: event_types.ClientConnectionBucketData): Promise { const updateFilter = this.updateDocFilter(data.user_id, data.client_id!); - await this.db.sdk_report_events.findOneAndUpdate( + await this.db.connection_report_events.findOneAndUpdate( updateFilter, { $set: data, @@ -63,7 +63,7 @@ export class MongoReportStorage implements storage.ReportStorage { } async reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise { const { connected_at, user_id, client_id } = data; - await this.db.sdk_report_events.findOneAndUpdate( + await this.db.connection_report_events.findOneAndUpdate( { client_id, user_id, @@ -81,7 +81,7 @@ export class MongoReportStorage implements storage.ReportStorage { } async getConnectedClients(data: event_types.ClientConnectionsRequest): Promise { const timeframeFilter = this.listConnectionsDateRange(data); - const result = await this.db.sdk_report_events + const result = await this.db.connection_report_events .aggregate([ { $match: { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts index d3cf0d71b..118eeae35 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts @@ -11,7 +11,7 @@ export const MongoTestReportStorageFactoryGenerator = (factoryOptions: MongoTest return async (options?: TestStorageOptions) => { const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); - await db.createSdkReportingCollection(); + await db.createConnectionReportingCollection(); if (!options?.doNotClear) { await db.clear(); diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 49c3eaa9a..012319f7f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -38,7 +38,7 @@ export class PowerSyncMongo { readonly locks: mongo.Collection; readonly bucket_state: mongo.Collection; readonly checkpoint_events: mongo.Collection; - readonly sdk_report_events: mongo.Collection; + readonly connection_report_events: mongo.Collection; readonly client: mongo.MongoClient; readonly db: mongo.Db; @@ -63,7 +63,7 @@ export class PowerSyncMongo { this.locks = this.db.collection('locks'); this.bucket_state = this.db.collection('bucket_state'); this.checkpoint_events = this.db.collection('checkpoint_events'); - this.sdk_report_events = this.db.collection('sdk_report_events'); + this.connection_report_events = this.db.collection('connection_report_events'); } /** @@ -81,7 +81,7 @@ export class PowerSyncMongo { await this.locks.deleteMany({}); await this.bucket_state.deleteMany({}); await this.custom_write_checkpoints.deleteMany({}); - await this.sdk_report_events.deleteMany({}); + await this.connection_report_events.deleteMany({}); } /** @@ -135,15 +135,15 @@ export class PowerSyncMongo { /** * Only use in migrations and tests. */ - async createSdkReportingCollection() { + async createConnectionReportingCollection() { const existingCollections = await this.db - .listCollections({ name: 'sdk_report_events' }, { nameOnly: false }) + .listCollections({ name: 'connection_report_events' }, { nameOnly: false }) .toArray(); const collection = existingCollections[0]; if (collection != null) { return; } - await this.db.createCollection('sdk_report_events'); + await this.db.createCollection('connection_report_events'); } } diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap new file mode 100644 index 000000000..d2b2dc247 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap @@ -0,0 +1,198 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`SDK reporting storage > Should create a connection report if its after a day 1`] = ` +[ + { + "client_id": "client_week", + "sdk": "powersync-js/1.24.5", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_week", + }, + { + "client_id": "client_week", + "sdk": "powersync-js/1.24.5", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_week", + }, +] +`; + +exports[`SDK reporting storage > Should delete rows older than specified range 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, + }, + ], + "users": 5, +} +`; + +exports[`SDK reporting storage > Should show connected users with start range 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + ], + "users": 1, +} +`; + +exports[`SDK reporting storage > Should show connected users with start range and end range 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + ], + "users": 1, +} +`; + +exports[`SDK reporting storage > Should show connection report data for user over the past day 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + ], + "users": 3, +} +`; + +exports[`SDK reporting storage > Should show connection report data for user over the past month 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.23.6", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.23.7", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, + }, + ], + "users": 7, +} +`; + +exports[`SDK reporting storage > Should show connection report data for user over the past week 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, + }, + ], + "users": 5, +} +`; + +exports[`SDK reporting storage > Should update a connected connection report and make it disconnected 1`] = ` +[ + { + "client_id": "client_three", + "sdk": "powersync-js/1.21.2", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_three", + }, +] +`; + +exports[`SDK reporting storage > Should update a connection report if its within a day 1`] = ` +[ + { + "client_id": "client_one", + "sdk": "powersync-dart/1.6.4", + "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android", + "user_id": "user_one", + }, +] +`; diff --git a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts b/modules/module-mongodb-storage/test/src/connection-report-storage.test.ts similarity index 88% rename from modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts rename to modules/module-mongodb-storage/test/src/connection-report-storage.test.ts index 68cfd17ff..b8b8e6c96 100644 --- a/modules/module-mongodb-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-mongodb-storage/test/src/connection-report-storage.test.ts @@ -96,7 +96,7 @@ describe('SDK reporting storage', async () => { }; async function loadData() { - await factory.db.sdk_report_events.insertMany([ + await factory.db.connection_report_events.insertMany([ user_one, user_two, user_three, @@ -108,7 +108,7 @@ describe('SDK reporting storage', async () => { } function deleteData() { - return factory.db.sdk_report_events.deleteMany(); + return factory.db.connection_report_events.deleteMany(); } beforeAll(async () => { @@ -147,21 +147,21 @@ describe('SDK reporting storage', async () => { }); expect(current).toMatchSnapshot(); }); - it('Should show SDK scrape data for user over the past month', async () => { + it('Should show connection report data for user over the past month', async () => { const sdk = await factory.getClientConnectionReports({ start: monthAgo, end: now }); expect(sdk).toMatchSnapshot(); }); - it('Should show SDK scrape data for user over the past week', async () => { + it('Should show connection report data for user over the past week', async () => { const sdk = await factory.getClientConnectionReports({ start: weekAgo, end: now }); expect(sdk).toMatchSnapshot(); }); - it('Should show SDK scrape data for user over the past day', async () => { + it('Should show connection report data for user over the past day', async () => { const sdk = await factory.getClientConnectionReports({ start: dayAgo, end: now @@ -169,7 +169,7 @@ describe('SDK reporting storage', async () => { expect(sdk).toMatchSnapshot(); }); - it('Should update a sdk event if its within a day', async () => { + it('Should update a connection report if its within a day', async () => { const newConnectAt = new Date( now.getFullYear(), now.getMonth(), @@ -187,7 +187,7 @@ describe('SDK reporting storage', async () => { user_agent: user_one.user_agent }); - const sdk = await factory.db.sdk_report_events.find({ user_id: user_one.user_id }).toArray(); + const sdk = await factory.db.connection_report_events.find({ user_id: user_one.user_id }).toArray(); expect(sdk).toHaveLength(1); expect(new Date(sdk[0].connected_at)).toEqual(newConnectAt); expect(new Date(sdk[0].jwt_exp!)).toEqual(jwtExp); @@ -196,7 +196,7 @@ describe('SDK reporting storage', async () => { expect(cleaned).toMatchSnapshot(); }); - it('Should update a connected sdk event and make it disconnected', async () => { + it('Should update a connected connection report and make it disconnected', async () => { const disconnectAt = new Date( now.getFullYear(), now.getMonth(), @@ -215,14 +215,14 @@ describe('SDK reporting storage', async () => { connected_at: user_three.connected_at }); - const sdk = await factory.db.sdk_report_events.find({ user_id: user_three.user_id }).toArray(); + const sdk = await factory.db.connection_report_events.find({ user_id: user_three.user_id }).toArray(); expect(sdk).toHaveLength(1); expect(new Date(sdk[0].disconnected_at!)).toEqual(disconnectAt); const cleaned = removeVolatileFields(sdk); expect(cleaned).toMatchSnapshot(); }); - it('Should create a sdk event if its after a day', async () => { + it('Should create a connection report if its after a day', async () => { const newConnectAt = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, now.getHours()); const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); @@ -235,7 +235,7 @@ describe('SDK reporting storage', async () => { user_agent: user_week.user_agent }); - const sdk = await factory.db.sdk_report_events.find({ user_id: user_week.user_id }).toArray(); + const sdk = await factory.db.connection_report_events.find({ user_id: user_week.user_id }).toArray(); expect(sdk).toHaveLength(2); const cleaned = removeVolatileFields(sdk); expect(cleaned).toMatchSnapshot(); diff --git a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts index 77850ba02..54b54a57e 100644 --- a/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts +++ b/modules/module-postgres-storage/src/migrations/scripts/1684951997326-init.ts @@ -129,7 +129,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { ); `.execute(); await db.sql` - CREATE TABLE sdk_report_events ( + CREATE TABLE connection_report_events ( id TEXT PRIMARY KEY, user_agent TEXT NOT NULL, client_id TEXT NOT NULL, @@ -141,13 +141,15 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => { ) `.execute(); - await db.sql` CREATE INDEX sdk_list_index ON sdk_report_events (connected_at, jwt_exp, disconnected_at) `.execute(); + await db.sql` + CREATE INDEX sdk_list_index ON connection_report_events (connected_at, jwt_exp, disconnected_at) + `.execute(); - await db.sql`CREATE INDEX sdk_user_id_index ON sdk_report_events (user_id)`.execute(); + await db.sql`CREATE INDEX sdk_user_id_index ON connection_report_events (user_id)`.execute(); - await db.sql`CREATE INDEX sdk_client_id_index ON sdk_report_events (client_id)`.execute(); + await db.sql`CREATE INDEX sdk_client_id_index ON connection_report_events (client_id)`.execute(); - await db.sql`CREATE INDEX sdk_index ON sdk_report_events (sdk)`.execute(); + await db.sql`CREATE INDEX sdk_index ON connection_report_events (sdk)`.execute(); }); }; diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 683528c83..a1d9f6130 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -63,7 +63,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { SELECT * FROM - sdk_report_events + connection_report_events WHERE disconnected_at IS NULL AND jwt_exp > NOW() @@ -111,7 +111,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { SELECT * FROM - sdk_report_events + connection_report_events WHERE disconnected_at IS NULL AND jwt_exp > NOW() @@ -168,7 +168,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { const { gte, lt } = this.updateTableFilter(); const uuid = v4(); const result = await this.db.sql` - UPDATE sdk_report_events + UPDATE connection_report_events SET connected_at = ${{ type: 1184, value: connectIsoString }}, sdk = ${{ type: 'varchar', value: sdk }}, @@ -184,7 +184,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { if (result.results[1].status === 'UPDATE 0') { await this.db.sql` INSERT INTO - sdk_report_events ( + connection_report_events ( user_id, client_id, connected_at, @@ -211,7 +211,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { const disconnectIsoString = disconnected_at.toISOString(); const connectIsoString = connected_at.toISOString(); await this.db.sql` - UPDATE sdk_report_events + UPDATE connection_report_events SET disconnected_at = ${{ type: 1184, value: disconnectIsoString }}, jwt_exp = NULL @@ -236,7 +236,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { SELECT * FROM - sdk_report_events + connection_report_events WHERE connected_at >= ${{ type: 1184, value: start.toISOString() }} AND connected_at <= ${{ type: 1184, value: end.toISOString() }} @@ -278,7 +278,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { async deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise { const { date } = data; const result = await this.db.sql` - DELETE FROM sdk_report_events + DELETE FROM connection_report_events WHERE connected_at < ${{ type: 1184, value: date.toISOString() }} AND ( @@ -292,7 +292,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { const deletedRows = toInteger(result.results[1].status.split(' ')[1] || '0'); if (deletedRows > 0) { logger.info( - `TTL from ${date.toISOString()}: ${deletedRows} PostgresSQL rows have been removed from sdk_report_events.` + `TTL from ${date.toISOString()}: ${deletedRows} PostgresSQL rows have been removed from connection_report_events.` ); } } diff --git a/modules/module-postgres-storage/src/utils/db.ts b/modules/module-postgres-storage/src/utils/db.ts index 11ded913a..ce04143e5 100644 --- a/modules/module-postgres-storage/src/utils/db.ts +++ b/modules/module-postgres-storage/src/utils/db.ts @@ -23,6 +23,6 @@ export const dropTables = async (client: lib_postgres.DatabaseClient) => { await db.sql`DROP TABLE IF EXISTS custom_write_checkpoints`.execute(); await db.sql`DROP SEQUENCE IF EXISTS op_id_sequence`.execute(); await db.sql`DROP SEQUENCE IF EXISTS sync_rules_id_sequence`.execute(); - await db.sql`DROP TABLE IF EXISTS sdk_report_events`.execute(); + await db.sql`DROP TABLE IF EXISTS connection_report_events`.execute(); }); }; diff --git a/modules/module-postgres-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap new file mode 100644 index 000000000..3ddaf880a --- /dev/null +++ b/modules/module-postgres-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap @@ -0,0 +1,198 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`Connection report storage > Should create a sdk event if its after a day 1`] = ` +[ + { + "client_id": "client_week", + "sdk": "powersync-js/1.24.5", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_week", + }, + { + "client_id": "client_week", + "sdk": "powersync-js/1.24.5", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_week", + }, +] +`; + +exports[`Connection report storage > Should delete rows older than specified range 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, + }, + ], + "users": 5, +} +`; + +exports[`Connection report storage > Should show connected users with start range 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + ], + "users": 1, +} +`; + +exports[`Connection report storage > Should show connected users with start range and end range 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + ], + "users": 1, +} +`; + +exports[`Connection report storage > Should show connection report data for user over the past day 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + ], + "users": 3, +} +`; + +exports[`Connection report storage > Should show connection report data for user over the past month 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.23.6", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.23.7", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, + }, + ], + "users": 7, +} +`; + +exports[`Connection report storage > Should show connection report data for user over the past week 1`] = ` +{ + "sdks": [ + { + "clients": 1, + "sdk": "powersync-dart/1.6.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.1", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.2", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.21.4", + "users": 1, + }, + { + "clients": 1, + "sdk": "powersync-js/1.24.5", + "users": 1, + }, + ], + "users": 5, +} +`; + +exports[`Connection report storage > Should update a connected sdk event and make it disconnected 1`] = ` +[ + { + "client_id": "client_three", + "sdk": "powersync-js/1.21.2", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_three", + }, +] +`; + +exports[`Connection report storage > Should update a sdk event if its within a day 1`] = ` +[ + { + "client_id": "client_one", + "sdk": "powersync-dart/1.6.4", + "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android", + "user_id": "user_one", + }, +] +`; diff --git a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts b/modules/module-postgres-storage/test/src/connection-report-storage.test.ts similarity index 90% rename from modules/module-postgres-storage/test/src/sdk-report-storage.test.ts rename to modules/module-postgres-storage/test/src/connection-report-storage.test.ts index 38efdd37e..ffa813b4d 100644 --- a/modules/module-postgres-storage/test/src/sdk-report-storage.test.ts +++ b/modules/module-postgres-storage/test/src/connection-report-storage.test.ts @@ -11,7 +11,7 @@ function removeVolatileFields(sdks: event_types.ClientConnection[]): Partial { +describe('Connection report storage', async () => { const factory = await POSTGRES_REPORT_STORAGE_FACTORY(); const now = new Date(); const nowAdd5minutes = new Date( @@ -103,7 +103,7 @@ describe('SDK reporting storage', async () => { async function loadData() { await factory.db.sql` INSERT INTO - sdk_report_events ( + connection_report_events ( user_id, client_id, connected_at, @@ -188,7 +188,7 @@ describe('SDK reporting storage', async () => { } function deleteData() { - return factory.db.sql`TRUNCATE TABLE sdk_report_events`.execute(); + return factory.db.sql`TRUNCATE TABLE connection_report_events`.execute(); } beforeAll(async () => { @@ -227,21 +227,21 @@ describe('SDK reporting storage', async () => { }); expect(current).toMatchSnapshot(); }); - it('Should show SDK scrape data for user over the past month', async () => { + it('Should show connection report data for user over the past month', async () => { const sdk = await factory.getClientConnectionReports({ start: monthAgo, end: now }); expect(sdk).toMatchSnapshot(); }); - it('Should show SDK scrape data for user over the past week', async () => { + it('Should show connection report data for user over the past week', async () => { const sdk = await factory.getClientConnectionReports({ start: weekAgo, end: now }); expect(sdk).toMatchSnapshot(); }); - it('Should show SDK scrape data for user over the past day', async () => { + it('Should show connection report data for user over the past day', async () => { const sdk = await factory.getClientConnectionReports({ start: dayAgo, end: now @@ -249,7 +249,7 @@ describe('SDK reporting storage', async () => { expect(sdk).toMatchSnapshot(); }); - it('Should update a sdk event if its within a day', async () => { + it('Should update a connection event if its within a day', async () => { const newConnectAt = new Date( now.getFullYear(), now.getMonth(), @@ -268,7 +268,7 @@ describe('SDK reporting storage', async () => { }); const sdk = await factory.db - .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_one.user_id }}`.rows(); + .sql`SELECT * FROM connection_report_events WHERE user_id = ${{ type: 'varchar', value: user_one.user_id }}`.rows(); expect(sdk).toHaveLength(1); expect(new Date(sdk[0].connected_at).toISOString()).toEqual(newConnectAt.toISOString()); expect(new Date(sdk[0].jwt_exp!).toISOString()).toEqual(jwtExp.toISOString()); @@ -277,7 +277,7 @@ describe('SDK reporting storage', async () => { expect(cleaned).toMatchSnapshot(); }); - it('Should update a connected sdk event and make it disconnected', async () => { + it('Should update a connection event and make it disconnected', async () => { const disconnectAt = new Date( now.getFullYear(), now.getMonth(), @@ -297,14 +297,14 @@ describe('SDK reporting storage', async () => { }); const sdk = await factory.db - .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_three.user_id }}`.rows(); + .sql`SELECT * FROM connection_report_events WHERE user_id = ${{ type: 'varchar', value: user_three.user_id }}`.rows(); expect(sdk).toHaveLength(1); expect(new Date(sdk[0].disconnected_at!).toISOString()).toEqual(disconnectAt.toISOString()); const cleaned = removeVolatileFields(sdk); expect(cleaned).toMatchSnapshot(); }); - it('Should create a sdk event if its after a day', async () => { + it('Should create a connection event if its after a day', async () => { const newConnectAt = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, now.getHours()); const jwtExp = new Date(newConnectAt.getFullYear(), newConnectAt.getMonth(), newConnectAt.getDate() + 1); @@ -318,7 +318,7 @@ describe('SDK reporting storage', async () => { }); const sdk = await factory.db - .sql`SELECT * FROM sdk_report_events WHERE user_id = ${{ type: 'varchar', value: user_week.user_id }}`.rows(); + .sql`SELECT * FROM connection_report_events WHERE user_id = ${{ type: 'varchar', value: user_week.user_id }}`.rows(); expect(sdk).toHaveLength(2); const cleaned = removeVolatileFields(sdk); expect(cleaned).toMatchSnapshot(); From 9e9e91b4d741ece726f1c91222c99d495a6bc80c Mon Sep 17 00:00:00 2001 From: JuanB Date: Wed, 20 Aug 2025 12:50:12 +0200 Subject: [PATCH 163/169] fixed tests --- .../sdk-report-storage.test.ts.snap | 198 ------------------ .../connection-report-storage.test.ts.snap | 39 ++++ .../sdk-report-storage.test.ts.snap | 198 ------------------ 3 files changed, 39 insertions(+), 396 deletions(-) delete mode 100644 modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap delete mode 100644 modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap deleted file mode 100644 index 77ab95509..000000000 --- a/modules/module-mongodb-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap +++ /dev/null @@ -1,198 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`SDK reporting storage > Should create a sdk event if its after a day 1`] = ` -[ - { - "client_id": "client_week", - "sdk": "powersync-js/1.24.5", - "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", - "user_id": "user_week", - }, - { - "client_id": "client_week", - "sdk": "powersync-js/1.24.5", - "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", - "user_id": "user_week", - }, -] -`; - -exports[`SDK reporting storage > Should delete rows older than specified range 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-dart/1.6.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.1", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.2", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.24.5", - "users": 1, - }, - ], - "users": 5, -} -`; - -exports[`SDK reporting storage > Should show SDK scrape data for user over the past day 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-dart/1.6.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.1", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.4", - "users": 1, - }, - ], - "users": 3, -} -`; - -exports[`SDK reporting storage > Should show SDK scrape data for user over the past month 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-dart/1.6.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.1", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.2", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.23.6", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.23.7", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.24.5", - "users": 1, - }, - ], - "users": 7, -} -`; - -exports[`SDK reporting storage > Should show SDK scrape data for user over the past week 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-dart/1.6.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.1", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.2", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.24.5", - "users": 1, - }, - ], - "users": 5, -} -`; - -exports[`SDK reporting storage > Should show connected users with start range 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-dart/1.6.4", - "users": 1, - }, - ], - "users": 1, -} -`; - -exports[`SDK reporting storage > Should show connected users with start range and end range 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-js/1.21.1", - "users": 1, - }, - ], - "users": 1, -} -`; - -exports[`SDK reporting storage > Should update a connected sdk event and make it disconnected 1`] = ` -[ - { - "client_id": "client_three", - "sdk": "powersync-js/1.21.2", - "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", - "user_id": "user_three", - }, -] -`; - -exports[`SDK reporting storage > Should update a sdk event if its within a day 1`] = ` -[ - { - "client_id": "client_one", - "sdk": "powersync-dart/1.6.4", - "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android", - "user_id": "user_one", - }, -] -`; diff --git a/modules/module-postgres-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap index 3ddaf880a..04dfa692d 100644 --- a/modules/module-postgres-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap +++ b/modules/module-postgres-storage/test/src/__snapshots__/connection-report-storage.test.ts.snap @@ -1,5 +1,22 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html +exports[`Connection report storage > Should create a connection event if its after a day 1`] = ` +[ + { + "client_id": "client_week", + "sdk": "powersync-js/1.24.5", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_week", + }, + { + "client_id": "client_week", + "sdk": "powersync-js/1.24.5", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_week", + }, +] +`; + exports[`Connection report storage > Should create a sdk event if its after a day 1`] = ` [ { @@ -186,6 +203,28 @@ exports[`Connection report storage > Should update a connected sdk event and mak ] `; +exports[`Connection report storage > Should update a connection event and make it disconnected 1`] = ` +[ + { + "client_id": "client_three", + "sdk": "powersync-js/1.21.2", + "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", + "user_id": "user_three", + }, +] +`; + +exports[`Connection report storage > Should update a connection event if its within a day 1`] = ` +[ + { + "client_id": "client_one", + "sdk": "powersync-dart/1.6.4", + "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android", + "user_id": "user_one", + }, +] +`; + exports[`Connection report storage > Should update a sdk event if its within a day 1`] = ` [ { diff --git a/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap b/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap deleted file mode 100644 index 77ab95509..000000000 --- a/modules/module-postgres-storage/test/src/__snapshots__/sdk-report-storage.test.ts.snap +++ /dev/null @@ -1,198 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`SDK reporting storage > Should create a sdk event if its after a day 1`] = ` -[ - { - "client_id": "client_week", - "sdk": "powersync-js/1.24.5", - "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", - "user_id": "user_week", - }, - { - "client_id": "client_week", - "sdk": "powersync-js/1.24.5", - "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", - "user_id": "user_week", - }, -] -`; - -exports[`SDK reporting storage > Should delete rows older than specified range 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-dart/1.6.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.1", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.2", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.24.5", - "users": 1, - }, - ], - "users": 5, -} -`; - -exports[`SDK reporting storage > Should show SDK scrape data for user over the past day 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-dart/1.6.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.1", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.4", - "users": 1, - }, - ], - "users": 3, -} -`; - -exports[`SDK reporting storage > Should show SDK scrape data for user over the past month 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-dart/1.6.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.1", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.2", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.23.6", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.23.7", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.24.5", - "users": 1, - }, - ], - "users": 7, -} -`; - -exports[`SDK reporting storage > Should show SDK scrape data for user over the past week 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-dart/1.6.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.1", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.2", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.21.4", - "users": 1, - }, - { - "clients": 1, - "sdk": "powersync-js/1.24.5", - "users": 1, - }, - ], - "users": 5, -} -`; - -exports[`SDK reporting storage > Should show connected users with start range 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-dart/1.6.4", - "users": 1, - }, - ], - "users": 1, -} -`; - -exports[`SDK reporting storage > Should show connected users with start range and end range 1`] = ` -{ - "sdks": [ - { - "clients": 1, - "sdk": "powersync-js/1.21.1", - "users": 1, - }, - ], - "users": 1, -} -`; - -exports[`SDK reporting storage > Should update a connected sdk event and make it disconnected 1`] = ` -[ - { - "client_id": "client_three", - "sdk": "powersync-js/1.21.2", - "user_agent": "powersync-js/1.21.0 powersync-web Firefox/141 linux", - "user_id": "user_three", - }, -] -`; - -exports[`SDK reporting storage > Should update a sdk event if its within a day 1`] = ` -[ - { - "client_id": "client_one", - "sdk": "powersync-dart/1.6.4", - "user_agent": "powersync-dart/1.6.4 Dart (flutter-web) Chrome/128 android", - "user_id": "user_one", - }, -] -`; From f98604975e02f51658fb01aeedd29a4d44c1c740 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 21 Aug 2025 09:15:59 +0200 Subject: [PATCH 164/169] removed index drop migrations --- .../migrations/1752661449910-connection-reporting.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-connection-reporting.ts b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-connection-reporting.ts index 6459bc7ef..a74c35291 100644 --- a/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-connection-reporting.ts +++ b/modules/module-mongodb-storage/src/migrations/db/migrations/1752661449910-connection-reporting.ts @@ -51,18 +51,6 @@ export const down: migrations.PowerSyncMigrationFunction = async (context) => { const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig); try { - if (await db.connection_report_events.indexExists('connection_list_index')) { - await db.connection_report_events.dropIndex('connection_list_index'); - } - if (await db.connection_report_events.indexExists('connection_user_id_index')) { - await db.connection_report_events.dropIndex('connection_user_id_index'); - } - if (await db.connection_report_events.indexExists('connection_client_id_index')) { - await db.connection_report_events.dropIndex('connection_client_id_index'); - } - if (await db.connection_report_events.indexExists('connection_index')) { - await db.connection_report_events.dropIndex('connection_index'); - } await db.db.dropCollection('connection_report_events'); } finally { await db.client.close(); From fe2ef90bd5a0c62258537ef132c7dafebcbd0f32 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 21 Aug 2025 15:06:59 +0200 Subject: [PATCH 165/169] removed unused type --- packages/types/src/reports.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/types/src/reports.ts b/packages/types/src/reports.ts index 51df86316..e7adc919a 100644 --- a/packages/types/src/reports.ts +++ b/packages/types/src/reports.ts @@ -4,7 +4,6 @@ export enum EventsEngineEventType { SDK_DELETE_OLD = 'sdk-delete-old' } -export type TimeFrames = 'hour' | 'day' | 'week' | 'month'; export type SubscribeEvents = { [EventsEngineEventType.SDK_CONNECT_EVENT]: ClientConnectionEventData; [EventsEngineEventType.SDK_DISCONNECT_EVENT]: ClientDisconnectionEventData; From 7a6673e9657045e71cd068275ee7a6036fe4e2c1 Mon Sep 17 00:00:00 2001 From: JuanB Date: Thu, 21 Aug 2025 16:12:26 +0200 Subject: [PATCH 166/169] changed date query --- .../storage/PostgresReportStorageFactory.ts | 47 +------------------ 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index a1d9f6130..215be68cb 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -56,53 +56,8 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { } private async listConnectionsQuery(data: event_types.ClientConnectionsRequest) { const { range } = data; - if (!range) { - return this.db.sql` - WITH - filtered AS ( - SELECT - * - FROM - connection_report_events - WHERE - disconnected_at IS NULL - AND jwt_exp > NOW() - ), - unique_users AS ( - SELECT - COUNT(DISTINCT user_id) AS count - FROM - filtered - ), - sdk_versions_array AS ( - SELECT - sdk, - COUNT(DISTINCT client_id) AS clients, - COUNT(DISTINCT user_id) AS users - FROM - filtered - GROUP BY - sdk - ) - SELECT - ( - SELECT - COALESCE(count, 0) - FROM - unique_users - ) AS users, - ( - SELECT - JSON_AGG(ROW_TO_JSON(s)) - FROM - sdk_versions_array s - ) AS sdks; - ` - .decoded(SdkReporting) - .first(); - } const endDate = data.range?.end ? new Date(data.range.end) : new Date(); - const startDate = new Date(range.start); + const startDate = !range ? new Date(0) : new Date(range.start); const lt = endDate.toISOString(); const gt = startDate.toISOString(); return await this.db.sql` From 4a0c4ca57dccc5c857f9b04f4e5fb32c1f10a64b Mon Sep 17 00:00:00 2001 From: JuanB Date: Fri, 22 Aug 2025 09:33:48 +0200 Subject: [PATCH 167/169] added comments --- .../src/storage/MongoReportStorage.ts | 10 +++--- .../storage/PostgresReportStorageFactory.ts | 12 ++++--- .../service-core/src/events/EventsEngine.ts | 8 ++--- .../src/routes/configure-fastify.ts | 1 - .../src/routes/endpoints/socket-route.ts | 3 +- .../src/routes/endpoints/sync-stream.ts | 3 +- packages/service-core/src/routes/router.ts | 6 ++-- .../service-core/src/storage/ReportStorage.ts | 32 +++++++++++++++++-- packages/types/src/reports.ts | 22 +++++++++++-- 9 files changed, 75 insertions(+), 22 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts index 02c2aac6b..6548a15ca 100644 --- a/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoReportStorage.ts @@ -30,10 +30,10 @@ export class MongoReportStorage implements storage.ReportStorage { async getClientConnectionReports( data: event_types.ClientConnectionReportRequest - ): Promise { + ): Promise { const { start, end } = data; const result = await this.db.connection_report_events - .aggregate([ + .aggregate([ { $match: { connected_at: { $lte: end, $gte: start } @@ -79,10 +79,12 @@ export class MongoReportStorage implements storage.ReportStorage { } ); } - async getConnectedClients(data: event_types.ClientConnectionsRequest): Promise { + async getConnectedClients( + data: event_types.ClientConnectionsRequest + ): Promise { const timeframeFilter = this.listConnectionsDateRange(data); const result = await this.db.connection_report_events - .aggregate([ + .aggregate([ { $match: { disconnected_at: { $exists: false }, diff --git a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts index 215be68cb..5f0e29bff 100644 --- a/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresReportStorageFactory.ts @@ -42,7 +42,9 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { }; } - private mapListCurrentConnectionsResponse(result: SdkReportingDecoded | null): event_types.ClientConnectionReport { + private mapListCurrentConnectionsResponse( + result: SdkReportingDecoded | null + ): event_types.ClientConnectionReportResponse { if (!result) { return { users: 0, @@ -119,7 +121,7 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { async reportClientConnection(data: event_types.ClientConnectionBucketData): Promise { const { sdk, connected_at, user_id, user_agent, jwt_exp, client_id } = data; const connectIsoString = connected_at.toISOString(); - const jwtExpIsoString = jwt_exp!.toISOString(); + const jwtExpIsoString = jwt_exp.toISOString(); const { gte, lt } = this.updateTableFilter(); const uuid = v4(); const result = await this.db.sql` @@ -176,14 +178,16 @@ export class PostgresReportStorageFactory implements storage.ReportStorage { AND connected_at = ${{ type: 1184, value: connectIsoString }} `.execute(); } - async getConnectedClients(data: event_types.ClientConnectionsRequest): Promise { + async getConnectedClients( + data: event_types.ClientConnectionsRequest + ): Promise { const result = await this.listConnectionsQuery(data); return this.mapListCurrentConnectionsResponse(result); } async getClientConnectionReports( data: event_types.ClientConnectionReportRequest - ): Promise { + ): Promise { const { start, end } = data; const result = await this.db.sql` WITH diff --git a/packages/service-core/src/events/EventsEngine.ts b/packages/service-core/src/events/EventsEngine.ts index f67dbcb12..61cea555e 100644 --- a/packages/service-core/src/events/EventsEngine.ts +++ b/packages/service-core/src/events/EventsEngine.ts @@ -12,6 +12,10 @@ export class EventsEngine { }); } + /** + * All new events added need to be subscribed to be used. + * @example engine.subscribe(new MyNewEvent(storageEngine)); + */ subscribe(event: event_types.EmitterEvent): void { if (!this.events.has(event.event)) { this.events.add(event.event); @@ -23,10 +27,6 @@ export class EventsEngine { return Array.from(this.events.values()); } - countListeners(eventName: event_types.EventsEngineEventType): number { - return this.emitter.listenerCount(eventName); - } - emit(event: K, data: event_types.SubscribeEvents[K]): void { this.emitter.emit(event, data); } diff --git a/packages/service-core/src/routes/configure-fastify.ts b/packages/service-core/src/routes/configure-fastify.ts index ba8906d15..45c5eacc2 100644 --- a/packages/service-core/src/routes/configure-fastify.ts +++ b/packages/service-core/src/routes/configure-fastify.ts @@ -1,5 +1,4 @@ import type fastify from 'fastify'; -import * as uuid from 'uuid'; import { registerFastifyNotFoundHandler, registerFastifyRoutes } from './route-register.js'; diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 4b0e3ed59..e6d46eb73 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -26,7 +26,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => client_id: params.client_id, user_id: context.user_id!, user_agent: context.user_agent, - jwt_exp: context.token_payload?.exp ? new Date(context.token_payload.exp * 1000) : undefined, + // At this point the token_payload is guaranteed to be present + jwt_exp: new Date(context.token_payload!.exp * 1000), connected_at: new Date(streamStart) }; diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index 5ff5f78eb..580fb429c 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -45,7 +45,8 @@ export const syncStreamed = routeDefinition({ client_id: clientId, user_id: payload.context.user_id!, user_agent: userAgent as string, - jwt_exp: token_payload?.exp ? new Date(token_payload?.exp * 1000) : undefined, + // At this point the token_payload is guaranteed to be present + jwt_exp: new Date(token_payload!.exp * 1000), connected_at: new Date(streamStart) }; diff --git a/packages/service-core/src/routes/router.ts b/packages/service-core/src/routes/router.ts index 3298c6e6b..9ee9d075b 100644 --- a/packages/service-core/src/routes/router.ts +++ b/packages/service-core/src/routes/router.ts @@ -1,4 +1,4 @@ -import { router, ServiceError, Logger } from '@powersync/lib-services-framework'; +import { Logger, router } from '@powersync/lib-services-framework'; import type { JwtPayload } from '../auth/auth-index.js'; import { ServiceContext } from '../system/ServiceContext.js'; import { RouterEngine } from './RouterEngine.js'; @@ -31,11 +31,11 @@ export type BasicRouterRequest = { hostname: string; }; -export type ConextProviderOptions = { +export type ContextProviderOptions = { logger: Logger; }; -export type ContextProvider = (request: BasicRouterRequest, options: ConextProviderOptions) => Promise; +export type ContextProvider = (request: BasicRouterRequest, options: ContextProviderOptions) => Promise; export type RequestEndpoint< I, diff --git a/packages/service-core/src/storage/ReportStorage.ts b/packages/service-core/src/storage/ReportStorage.ts index df6d5838c..c845adb15 100644 --- a/packages/service-core/src/storage/ReportStorage.ts +++ b/packages/service-core/src/storage/ReportStorage.ts @@ -1,11 +1,39 @@ import { event_types } from '@powersync/service-types'; +/** + * Represents a configured report storage. + * + * Report storage is used for storing localized data for the instance. + * Data can then be used for reporting purposes. + * + */ export interface ReportStorage extends AsyncDisposable { + /** + * Report a client connection. + */ reportClientConnection(data: event_types.ClientConnectionBucketData): Promise; + /** + * Report a client disconnection. + */ reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise; - getConnectedClients(data: event_types.ClientConnectionsRequest): Promise; + /** + * Get currently connected clients. + * This will return any short or long term connected clients. + * Clients that have no disconnected_at timestamp and that have a valid jwt_exp timestamp are considered connected. + */ + getConnectedClients(data: event_types.ClientConnectionsRequest): Promise; + /** + * Get a report of client connections over a day, week or month. + * This is internally used to generate reports over it always returns the previous day, week or month. + * Usually this is call on the start of the new day, week or month. It will return all unique completed connections + * as well as uniques currently connected clients. + */ getClientConnectionReports( data: event_types.ClientConnectionReportRequest - ): Promise; + ): Promise; + /** + * Delete old connection data based on a specific date. + * This is used to clean up old connection data that is no longer needed. + */ deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise; } diff --git a/packages/types/src/reports.ts b/packages/types/src/reports.ts index e7adc919a..9730763d6 100644 --- a/packages/types/src/reports.ts +++ b/packages/types/src/reports.ts @@ -4,13 +4,25 @@ export enum EventsEngineEventType { SDK_DELETE_OLD = 'sdk-delete-old' } +/** + * Events engine event types. + * Any new events will need to be added here with the data structure they expect. + */ export type SubscribeEvents = { [EventsEngineEventType.SDK_CONNECT_EVENT]: ClientConnectionEventData; [EventsEngineEventType.SDK_DISCONNECT_EVENT]: ClientDisconnectionEventData; [EventsEngineEventType.SDK_DELETE_OLD]: DeleteOldConnectionData; }; +/** + * Events handler functions + */ export type EventHandlerFunc = (data: SubscribeEvents[K]) => Promise | void; + +/** + * Emitter event interface. + * Create a class extending EmitterEvent and implement the handler function. + */ export interface EmitterEvent { event: K; handler: EventHandlerFunc; @@ -20,10 +32,14 @@ export type ConnectedUserData = { client_id?: string; user_id: string; user_agent?: string; - jwt_exp?: Date; + jwt_exp: Date; }; export type DeleteOldConnectionData = { + /** + * Date before which all connection data should be deleted. + * This is used to clean up old connection data that is no longer needed. + */ date: Date; }; @@ -33,6 +49,7 @@ export type ClientConnectionEventData = { export type ClientConnectionBucketData = { connected_at: Date; + /** parsed sdk version from the user agent. */ sdk: string; } & ConnectedUserData; @@ -41,6 +58,7 @@ export type ClientDisconnectionEventData = { connected_at: Date; } & ConnectedUserData; +/** client connection schema stored locally */ export type ClientConnection = { id?: string; sdk: string; @@ -52,7 +70,7 @@ export type ClientConnection = { disconnected_at?: Date; }; -export type ClientConnectionReport = { +export type ClientConnectionReportResponse = { users: number; sdks: { sdk: string; From acd4bea3ab555e6edcc8a9288db566ef791bbcfd Mon Sep 17 00:00:00 2001 From: JuanB Date: Fri, 22 Aug 2025 11:14:29 +0200 Subject: [PATCH 168/169] refactored test utils to its own file in postgres and mongo --- modules/module-mongodb-storage/src/index.ts | 1 + .../src/storage/MongoBucketStorage.ts | 2 +- .../implementation/MongoBucketBatch.ts | 2 +- .../implementation/MongoSyncBucketStorage.ts | 2 +- .../MongoTestReportStorageFactoryGenerator.ts | 22 -------- .../MongoTestStorageFactoryGenerator.ts | 28 ---------- .../storage/implementation/PersistedBatch.ts | 2 +- .../src/storage/storage-index.ts | 4 +- .../src/utils/test-utils.ts | 55 +++++++++++++++++++ .../{storage/implementation => utils}/util.ts | 18 +----- .../src/utils/utils-index.ts | 2 + .../module-mongodb-storage/test/src/util.ts | 8 +-- modules/module-mongodb/test/src/util.ts | 2 +- modules/module-mysql/test/src/util.ts | 2 +- .../src/storage/storage-index.ts | 1 - .../test-utils.ts} | 12 ++-- .../src/utils/utils-index.ts | 1 + .../module-postgres-storage/test/src/util.ts | 3 +- .../test/src/storage_combination.test.ts | 2 +- modules/module-postgres/test/src/util.ts | 6 +- 20 files changed, 83 insertions(+), 92 deletions(-) delete mode 100644 modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts delete mode 100644 modules/module-mongodb-storage/src/storage/implementation/MongoTestStorageFactoryGenerator.ts create mode 100644 modules/module-mongodb-storage/src/utils/test-utils.ts rename modules/module-mongodb-storage/src/{storage/implementation => utils}/util.ts (87%) create mode 100644 modules/module-mongodb-storage/src/utils/utils-index.ts rename modules/module-postgres-storage/src/{storage/PostgresTestStorageFactoryGenerator.ts => utils/test-utils.ts} (88%) diff --git a/modules/module-mongodb-storage/src/index.ts b/modules/module-mongodb-storage/src/index.ts index 54ac0f55c..91aa8161b 100644 --- a/modules/module-mongodb-storage/src/index.ts +++ b/modules/module-mongodb-storage/src/index.ts @@ -5,3 +5,4 @@ export * as storage from './storage/storage-index.js'; export * from './types/types.js'; export * as types from './types/types.js'; +export * as utils from './utils/utils-index.js'; diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 82bbdf0bd..43200cb8d 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -12,7 +12,7 @@ import { PowerSyncMongo } from './implementation/db.js'; import { SyncRuleDocument } from './implementation/models.js'; import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js'; import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js'; -import { generateSlotName } from './implementation/util.js'; +import { generateSlotName } from '../utils/util.js'; export class MongoBucketStorage extends BaseObserver diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 6952092c8..535a8e043 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -28,7 +28,7 @@ import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; -import { idPrefixFilter } from './util.js'; +import { idPrefixFilter } from '../../utils/util.js'; /** * 15MB diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 58e58b4dd..67804b4cb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -32,7 +32,7 @@ import { BucketDataDocument, BucketDataKey, BucketStateDocument, SourceKey, Sour import { MongoBucketBatch } from './MongoBucketBatch.js'; import { MongoCompactor } from './MongoCompactor.js'; import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; -import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from './util.js'; +import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; export class MongoSyncBucketStorage diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts deleted file mode 100644 index 118eeae35..000000000 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoTestReportStorageFactoryGenerator.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { TestStorageOptions } from '@powersync/service-core'; -import { connectMongoForTests } from './util.js'; -import { MongoReportStorage } from '../MongoReportStorage.js'; - -export type MongoTestStorageOptions = { - url: string; - isCI: boolean; -}; - -export const MongoTestReportStorageFactoryGenerator = (factoryOptions: MongoTestStorageOptions) => { - return async (options?: TestStorageOptions) => { - const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); - - await db.createConnectionReportingCollection(); - - if (!options?.doNotClear) { - await db.clear(); - } - - return new MongoReportStorage(db); - }; -}; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoTestStorageFactoryGenerator.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoTestStorageFactoryGenerator.ts deleted file mode 100644 index a7457b43e..000000000 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoTestStorageFactoryGenerator.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { TestStorageOptions } from '@powersync/service-core'; -import { MongoBucketStorage } from '../MongoBucketStorage.js'; -import { connectMongoForTests } from './util.js'; - -export type MongoTestStorageOptions = { - url: string; - isCI: boolean; -}; - -export const MongoTestStorageFactoryGenerator = (factoryOptions: MongoTestStorageOptions) => { - return async (options?: TestStorageOptions) => { - const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); - - // None of the tests insert data into this collection, so it was never created - if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) { - await db.db.createCollection('bucket_parameters'); - } - - // Full migrations are not currently run for tests, so we manually create this - await db.createCheckpointEventsCollection(); - - if (!options?.doNotClear) { - await db.clear(); - } - - return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }); - }; -}; diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index b319053d5..4f1f0d859 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -16,7 +16,7 @@ import { CurrentDataDocument, SourceKey } from './models.js'; -import { replicaIdToSubkey } from './util.js'; +import { replicaIdToSubkey } from '../../utils/util.js'; /** * Maximum size of operations we write in a single transaction. diff --git a/modules/module-mongodb-storage/src/storage/storage-index.ts b/modules/module-mongodb-storage/src/storage/storage-index.ts index 8c934dbc0..cfb1d4ad0 100644 --- a/modules/module-mongodb-storage/src/storage/storage-index.ts +++ b/modules/module-mongodb-storage/src/storage/storage-index.ts @@ -7,9 +7,9 @@ export * from './implementation/MongoPersistedSyncRulesContent.js'; export * from './implementation/MongoStorageProvider.js'; export * from './implementation/MongoSyncBucketStorage.js'; export * from './implementation/MongoSyncRulesLock.js'; -export * from './implementation/MongoTestStorageFactoryGenerator.js'; export * from './implementation/OperationBatch.js'; export * from './implementation/PersistedBatch.js'; -export * from './implementation/util.js'; +export * from '../utils/util.js'; export * from './MongoBucketStorage.js'; export * from './MongoReportStorage.js'; +export * as test_utils from '../utils/test-utils.js'; diff --git a/modules/module-mongodb-storage/src/utils/test-utils.ts b/modules/module-mongodb-storage/src/utils/test-utils.ts new file mode 100644 index 000000000..e6ac7f621 --- /dev/null +++ b/modules/module-mongodb-storage/src/utils/test-utils.ts @@ -0,0 +1,55 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { PowerSyncMongo } from '../storage/implementation/db.js'; +import { TestStorageOptions } from '@powersync/service-core'; +import { MongoReportStorage } from '../storage/MongoReportStorage.js'; +import { MongoBucketStorage } from '../storage/MongoBucketStorage.js'; + +export type MongoTestStorageOptions = { + url: string; + isCI: boolean; +}; + +export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) { + return async (options?: TestStorageOptions) => { + const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); + + // None of the tests insert data into this collection, so it was never created + if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) { + await db.db.createCollection('bucket_parameters'); + } + + // Full migrations are not currently run for tests, so we manually create this + await db.createCheckpointEventsCollection(); + + if (!options?.doNotClear) { + await db.clear(); + } + + return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }); + }; +} + +export function mongoTestReportStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) { + return async (options?: TestStorageOptions) => { + const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); + + await db.createConnectionReportingCollection(); + + if (!options?.doNotClear) { + await db.clear(); + } + + return new MongoReportStorage(db); + }; +} + +export const connectMongoForTests = (url: string, isCI: boolean) => { + // Short timeout for tests, to fail fast when the server is not available. + // Slightly longer timeouts for CI, to avoid arbitrary test failures + const client = new mongo.MongoClient(url, { + connectTimeoutMS: isCI ? 15_000 : 5_000, + socketTimeoutMS: isCI ? 15_000 : 5_000, + serverSelectionTimeoutMS: isCI ? 15_000 : 2_500 + }); + return new PowerSyncMongo(client); +}; diff --git a/modules/module-mongodb-storage/src/storage/implementation/util.ts b/modules/module-mongodb-storage/src/utils/util.ts similarity index 87% rename from modules/module-mongodb-storage/src/storage/implementation/util.ts rename to modules/module-mongodb-storage/src/utils/util.ts index 5c1fdaf13..bd78ce2df 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/util.ts +++ b/modules/module-mongodb-storage/src/utils/util.ts @@ -3,9 +3,7 @@ import * as crypto from 'crypto'; import * as uuid from 'uuid'; import { mongo } from '@powersync/lib-service-mongodb'; import { storage, utils } from '@powersync/service-core'; - -import { PowerSyncMongo } from './db.js'; -import { BucketDataDocument } from './models.js'; +import { BucketDataDocument } from '../storage/implementation/models.js'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; export function idPrefixFilter(prefix: Partial, rest: (keyof T)[]): mongo.Condition { @@ -104,20 +102,6 @@ export function replicaIdToSubkey(table: bson.ObjectId, id: storage.ReplicaId): } } -/** - * Helper for unit tests - */ -export const connectMongoForTests = (url: string, isCI: boolean) => { - // Short timeout for tests, to fail fast when the server is not available. - // Slightly longer timeouts for CI, to avoid arbitrary test failures - const client = new mongo.MongoClient(url, { - connectTimeoutMS: isCI ? 15_000 : 5_000, - socketTimeoutMS: isCI ? 15_000 : 5_000, - serverSelectionTimeoutMS: isCI ? 15_000 : 2_500 - }); - return new PowerSyncMongo(client); -}; - export function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp) { // This is a workaround for the lack of direct support for snapshot reads in the MongoDB driver. if (!session.snapshotEnabled) { diff --git a/modules/module-mongodb-storage/src/utils/utils-index.ts b/modules/module-mongodb-storage/src/utils/utils-index.ts new file mode 100644 index 000000000..a42a9024d --- /dev/null +++ b/modules/module-mongodb-storage/src/utils/utils-index.ts @@ -0,0 +1,2 @@ +export * as test_utils from './test-utils.js'; +export * from './util.js'; diff --git a/modules/module-mongodb-storage/test/src/util.ts b/modules/module-mongodb-storage/test/src/util.ts index ae5672c57..4a7174056 100644 --- a/modules/module-mongodb-storage/test/src/util.ts +++ b/modules/module-mongodb-storage/test/src/util.ts @@ -1,14 +1,12 @@ import { env } from './env.js'; +import { mongoTestReportStorageFactoryGenerator, mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js'; -import { MongoTestStorageFactoryGenerator } from '@module/storage/implementation/MongoTestStorageFactoryGenerator.js'; -import { MongoTestReportStorageFactoryGenerator } from '@module/storage/implementation/MongoTestReportStorageFactoryGenerator.js'; - -export const INITIALIZED_MONGO_STORAGE_FACTORY = MongoTestStorageFactoryGenerator({ +export const INITIALIZED_MONGO_STORAGE_FACTORY = mongoTestStorageFactoryGenerator({ url: env.MONGO_TEST_URL, isCI: env.CI }); -export const INITIALIZED_MONGO_REPORT_STORAGE_FACTORY = MongoTestReportStorageFactoryGenerator({ +export const INITIALIZED_MONGO_REPORT_STORAGE_FACTORY = mongoTestReportStorageFactoryGenerator({ url: env.MONGO_TEST_URL, isCI: env.CI }); diff --git a/modules/module-mongodb/test/src/util.ts b/modules/module-mongodb/test/src/util.ts index db02a5158..980fcbb12 100644 --- a/modules/module-mongodb/test/src/util.ts +++ b/modules/module-mongodb/test/src/util.ts @@ -14,7 +14,7 @@ export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({ uri: TEST_URI }); -export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageFactoryGenerator({ +export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({ url: env.MONGO_TEST_URL, isCI: env.CI }); diff --git a/modules/module-mysql/test/src/util.ts b/modules/module-mysql/test/src/util.ts index cb72b12b4..509e0c557 100644 --- a/modules/module-mysql/test/src/util.ts +++ b/modules/module-mysql/test/src/util.ts @@ -19,7 +19,7 @@ export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({ uri: TEST_URI }); -export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageFactoryGenerator({ +export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({ url: env.MONGO_TEST_URL, isCI: env.CI }); diff --git a/modules/module-postgres-storage/src/storage/storage-index.ts b/modules/module-postgres-storage/src/storage/storage-index.ts index b97b6a966..4f8558818 100644 --- a/modules/module-postgres-storage/src/storage/storage-index.ts +++ b/modules/module-postgres-storage/src/storage/storage-index.ts @@ -2,4 +2,3 @@ export * from './PostgresBucketStorageFactory.js'; export * from './PostgresCompactor.js'; export * from './PostgresStorageProvider.js'; export * from './PostgresSyncRulesStorage.js'; -export * from './PostgresTestStorageFactoryGenerator.js'; diff --git a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts b/modules/module-postgres-storage/src/utils/test-utils.ts similarity index 88% rename from modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts rename to modules/module-postgres-storage/src/utils/test-utils.ts index 37f687200..91ee8acfc 100644 --- a/modules/module-postgres-storage/src/storage/PostgresTestStorageFactoryGenerator.ts +++ b/modules/module-postgres-storage/src/utils/test-utils.ts @@ -1,8 +1,8 @@ import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOptions } from '@powersync/service-core'; import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'; import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js'; -import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; -import { PostgresReportStorageFactory } from './PostgresReportStorageFactory.js'; +import { PostgresReportStorageFactory } from '../storage/PostgresReportStorageFactory.js'; +import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js'; export type PostgresTestStorageOptions = { url: string; @@ -13,7 +13,7 @@ export type PostgresTestStorageOptions = { migrationAgent?: (config: PostgresStorageConfigDecoded) => PostgresMigrationAgent; }; -export const postgresTestSetup = (factoryOptions: PostgresTestStorageOptions) => { +export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) { const BASE_CONFIG = { type: 'postgresql' as const, uri: factoryOptions.url, @@ -82,8 +82,8 @@ export const postgresTestSetup = (factoryOptions: PostgresTestStorageOptions) => }, migrate }; -}; +} -export const PostgresTestStorageFactoryGenerator = (factoryOptions: PostgresTestStorageOptions) => { +export function postgresTestStorageFactoryGenerator(factoryOptions: PostgresTestStorageOptions) { return postgresTestSetup(factoryOptions).factory; -}; +} diff --git a/modules/module-postgres-storage/src/utils/utils-index.ts b/modules/module-postgres-storage/src/utils/utils-index.ts index 65f808ff7..40370a4a7 100644 --- a/modules/module-postgres-storage/src/utils/utils-index.ts +++ b/modules/module-postgres-storage/src/utils/utils-index.ts @@ -2,3 +2,4 @@ export * from './bson.js'; export * from './bucket-data.js'; export * from './db.js'; export * from './ts-codec.js'; +export * as test_utils from './test-utils.js'; diff --git a/modules/module-postgres-storage/test/src/util.ts b/modules/module-postgres-storage/test/src/util.ts index 825be2ee1..d055dc343 100644 --- a/modules/module-postgres-storage/test/src/util.ts +++ b/modules/module-postgres-storage/test/src/util.ts @@ -1,7 +1,8 @@ import path from 'path'; import { fileURLToPath } from 'url'; -import { normalizePostgresStorageConfig, PostgresMigrationAgent, postgresTestSetup } from '../../src/index.js'; +import { normalizePostgresStorageConfig, PostgresMigrationAgent } from '../../src/index.js'; import { env } from './env.js'; +import { postgresTestSetup } from '../../src/utils/test-utils.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); diff --git a/modules/module-postgres/test/src/storage_combination.test.ts b/modules/module-postgres/test/src/storage_combination.test.ts index 1183d6806..89ec74de9 100644 --- a/modules/module-postgres/test/src/storage_combination.test.ts +++ b/modules/module-postgres/test/src/storage_combination.test.ts @@ -7,7 +7,7 @@ describe.skipIf(!env.TEST_POSTGRES_STORAGE)('replication storage combination - p test('should allow the same Postgres cluster to be used for data and storage', async () => { // Use the same cluster for the storage as the data source await using context = await WalStreamTestContext.open( - postgres_storage.PostgresTestStorageFactoryGenerator({ + postgres_storage.test_utils.postgresTestStorageFactoryGenerator({ url: env.PG_TEST_URL }), { doNotClear: false } diff --git a/modules/module-postgres/test/src/util.ts b/modules/module-postgres/test/src/util.ts index 130b70fe9..6367942af 100644 --- a/modules/module-postgres/test/src/util.ts +++ b/modules/module-postgres/test/src/util.ts @@ -2,7 +2,7 @@ import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js' import * as types from '@module/types/types.js'; import * as lib_postgres from '@powersync/lib-service-postgres'; import { logger } from '@powersync/lib-services-framework'; -import { BucketStorageFactory, InternalOpId, TestStorageFactory, TestStorageOptions } from '@powersync/service-core'; +import { BucketStorageFactory, InternalOpId, TestStorageFactory } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; import * as postgres_storage from '@powersync/service-module-postgres-storage'; @@ -11,12 +11,12 @@ import { describe, TestOptions } from 'vitest'; export const TEST_URI = env.PG_TEST_URL; -export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.MongoTestStorageFactoryGenerator({ +export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({ url: env.MONGO_TEST_URL, isCI: env.CI }); -export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTestStorageFactoryGenerator({ +export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.postgresTestStorageFactoryGenerator({ url: env.PG_STORAGE_TEST_URL }); From 084287c746c4d2583b3b6bb55c348235a85d9cd1 Mon Sep 17 00:00:00 2001 From: JuanB Date: Fri, 22 Aug 2025 11:25:09 +0200 Subject: [PATCH 169/169] oops --- modules/module-mongodb/test/src/util.ts | 2 +- modules/module-mysql/test/src/util.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb/test/src/util.ts b/modules/module-mongodb/test/src/util.ts index 980fcbb12..cda52142e 100644 --- a/modules/module-mongodb/test/src/util.ts +++ b/modules/module-mongodb/test/src/util.ts @@ -19,7 +19,7 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoT isCI: env.CI }); -export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTestStorageFactoryGenerator({ +export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.postgresTestStorageFactoryGenerator({ url: env.PG_STORAGE_TEST_URL }); diff --git a/modules/module-mysql/test/src/util.ts b/modules/module-mysql/test/src/util.ts index 509e0c557..4f18cdc53 100644 --- a/modules/module-mysql/test/src/util.ts +++ b/modules/module-mysql/test/src/util.ts @@ -24,7 +24,7 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoT isCI: env.CI }); -export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.PostgresTestStorageFactoryGenerator({ +export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.postgresTestStorageFactoryGenerator({ url: env.PG_STORAGE_TEST_URL });