🆕 new custom json stores system for custom data sharing / persistance…

… across apps

library/CushyStudio/examples/tutorial/widgets/example-custom-store.ts

@@ -0,0 +1,28 @@
+app({
+    ui: (ui) => ({
+        help: ui.markdown({
+            label: false,
+            markdown: [
+                //
+                `This example showcase how apps can remember data across`,
+                `multiple runs though custom data store.`,
+                `- Press play to see the amount of times you've pressed play`,
+                `- 📝 note: you can restart the app, update it, it will remember how many times you pressed it.`,
+            ].join('\n\n'),
+        }),
+    }),
+
+    run: async (sdk, ui) => {
+        const store = sdk.getStore_orCreateIfMissing('example-key-1337', () => ({ count: 0 }))
+        const prevValue = store.get()
+        sdk.output_Markdown(
+            [
+                //
+                `store created: **${sdk.formatAsRelativeDateTime(store.createdAt)}**`,
+                `previous run : **${sdk.formatAsRelativeDateTime(store.updatedAt)}**`,
+                `run count    : **${store.get().count + 1} times**`,
+            ].join('\n\n'),
+        )
+        store.update({ json: { count: prevValue.count + 1 } })
+    },
+})

src/app/layout/CreateDeckModalUI.tsx

@@ -9,7 +9,7 @@ export class CreateDeckModalState {
         makeAutoObservable(this)
     }
     ref = createRef<HTMLDialogElement>()
-    deckName: string = 'my-cushy-deck'
+    deckName: string = 'cushy-package'
     open = false
     isCreating = false
     error: Maybe<string>

src/back/Runtime.ts

@@ -30,6 +30,8 @@ import { InvalidPromptError } from './RuntimeError'
 import { Status } from './Status'
 
 import child_process from 'child_process'
+import { CustomDataL } from 'src/models/CustomData'
+import { _formatAsRelativeDateTime } from 'src/updater/_getRelativeTimeString'
 
 export type ImageAndMask = HasSingle_IMAGE & HasSingle_MASK
 
@@ -103,6 +105,17 @@ export class Runtime<FIELDS extends WidgetDict = any> {
      * */
     formInstance!: Widget_group<FIELDS>
 
+    getStore_orCrashIfMissing = <T>(key: string): CustomDataL<T> => {
+        return this.st.db.custom_datas.getOrThrow(key)
+    }
+
+    getStore_orCreateIfMissing = <T>(key: string, def: () => T): CustomDataL<T> => {
+        return this.st.db.custom_datas.getOrCreate(key, () => ({
+            id: key,
+            json: def(),
+        }))
+    }
+
     executeDraft = async (draftID: DraftID, args: any) => {
         throw new Error('🔴 not yet implemented')
     }
@@ -210,8 +223,12 @@ export class Runtime<FIELDS extends WidgetDict = any> {
     /** check if the current connected ComfyUI backend has a given checkpoint */
     hasCheckpoint = (loraName: string): boolean => this.schema.hasLora(loraName)
 
-    /** run an imagemagick convert action */
-    imagemagicConvert = (
+    /**
+     * helper function to quickly run some imagemagick convert command
+     * on an existing MediaImage instance, regardless of it's provenance
+     * 🔶 works but unfinished
+     */
+    exec_imagemagickConvert = (
         //
         img: MediaImageL,
         partialCmd: string,
@@ -261,19 +278,16 @@ export class Runtime<FIELDS extends WidgetDict = any> {
 
     // ------------------------------------------------------------------------------------
 
+    /** outputs a gaussian splat asset, accessible at the given URL */
     output_GaussianSplat = (p: { url: string }) => {
         this.st.db.media_splats.create({
             url: p.url,
             stepID: this.step.id,
         })
     }
+
     /** output a 3d scene from an image and its displacement and depth maps */
-    output_3dImage = (p: {
-        //
-        image: string
-        depth: string
-        normal: string
-    }) => {
+    output_3dImage = (p: { image: string; depth: string; normal: string }) => {
         const image = this.generatedImages.find((i) => i.filename.startsWith(p.image))
         const depth = this.generatedImages.find((i) => i.filename.startsWith(p.depth))
         const normal = this.generatedImages.find((i) => i.filename.startsWith(p.normal))
@@ -289,32 +303,9 @@ export class Runtime<FIELDS extends WidgetDict = any> {
             normalMap: normal.url,
             stepID: this.step.id,
         })
-        console.log('🟢🟢🟢🟢🟢🟢 displaced')
-        // this.st.layout.FOCUS_OR_CREATE('DisplacedImage', {
-        //     width: image.data.width ?? 512,
-        //     height: image.data.height ?? 512,
-        //     image: image.url,
-        //     depthMap: depth.url,
-        //     normalMap: normal.url,
-        // })
     }
 
-    // ------------------------------------------------------------------------------------
-    // output_image = (p: { url: string }) => {
-    //     const img = this.st.db.images.create({
-    //         downloaded: true,
-    //         localFilePath: './foobbabbababa',
-    //         comfyImageInfo: { filename: 'test' },
-    //     })
-    //     this.st.layout.FOCUS_OR_CREATE('DisplacedImage', {
-    //         width: image.data.width ?? 512,
-    //         height: image.data.height ?? 512,
-    //         image: image.url,
-    //         depthMap: depth.url,
-    //         normalMap: normal.url,
-    //     })
-    // }
-
+    /** 🔴 unfinished */
     output_File = async (path: RelativePath, content: string): Promise<void> => {
         const absPath = this.st.resolve(this.folder, path)
         writeFileSync(absPath, content, 'utf-8')
@@ -329,18 +320,48 @@ export class Runtime<FIELDS extends WidgetDict = any> {
         })
     }
 
-    output_Markdown = (p: { title: string; markdownContent: string }) => {
-        this.st.db.media_texts.create({
-            kind: 'markdown',
-            title: p.title,
-            content: p.markdownContent,
+    output_Markdown = (p: string | { title: string; markdownContent: string }) => {
+        const title = typeof p === 'string' ? '<no-title>' : p.title
+        const content = typeof p === 'string' ? p : p.markdownContent
+        this.st.db.media_texts.create({ kind: 'markdown', title, content, stepID: this.step.id })
+    }
+
+    output_text = (p: { title: string; message: Printable } | string) => {
+        const [title, message] = typeof p === 'string' ? ['<no-title>', p] : [p.title, p.message]
+        let msg = this.extractString(message)
+        console.info(msg)
+        this.step.db.media_texts.create({
+            kind: 'text',
+            title: title,
+            content: msg,
             stepID: this.step.id,
         })
+    }
 
-        // const htmlContent = marked.parse(p.markdownContent)
-        // this.step.addOutput({ type: 'show-html', content: htmlContent, title: p.title })
-        // this.st.broadCastToAllClients({ type: 'show-html', content: htmlContent, title: p.title })
+    /**
+     * @deprecated
+     * use `output_text` instead;
+     * */
+    print = (message: Printable) => {
+        this.output_text({ title: '<no-title>', message })
     }
+
+    // ------------------------------------------------------------------------------------
+    // output_image = (p: { url: string }) => {
+    //     const img = this.st.db.images.create({
+    //         downloaded: true,
+    //         localFilePath: './foobbabbababa',
+    //         comfyImageInfo: { filename: 'test' },
+    //     })
+    //     this.st.layout.FOCUS_OR_CREATE('DisplacedImage', {
+    //         width: image.data.width ?? 512,
+    //         height: image.data.height ?? 512,
+    //         image: image.url,
+    //         depthMap: depth.url,
+    //         normalMap: normal.url,
+    //     })
+    // }
+
     // ===================================================================================================
 
     // private
@@ -442,6 +463,7 @@ ${ffmpegComandInfos.framesFileContent}
      * e.g.: "EasyNegative" => "embedding:EasyNegative"
      * */
     formatEmbeddingForComfyUI = (t: Embeddings) => `embedding:${t}`
+    formatAsRelativeDateTime = (date: Date | number): string => _formatAsRelativeDateTime(date)
 
     // 🐉 /** ask the user a few informations */
     // 🐉 ask: InfoRequestFn = async <const Req extends { [key: string]: Widget }>(
@@ -597,26 +619,6 @@ ${ffmpegComandInfos.framesFileContent}
         return `❌ (impossible to extract string from ${typeof message} / ${(message as any)?.constructor?.name})`
     }
 
-    /**
-     * @deprecated
-     * use `output_text` instead;
-     * */
-    print = (message: Printable) => {
-        this.output_text({ title: '<no-title>', message })
-    }
-
-    output_text = (p: { title: string; message: Printable } | string) => {
-        const [title, message] = typeof p === 'string' ? ['<no-title>', p] : [p.title, p.message]
-        let msg = this.extractString(message)
-        console.info(msg)
-        this.step.db.media_texts.create({
-            kind: 'text',
-            title: title,
-            content: msg,
-            stepID: this.step.id,
-        })
-    }
-
     /** upload a file from disk to the ComfyUI backend */
     // uploadImgFromDisk = async (path: string): Promise<ComfyUploadImageResult> => {
     //     return this.workspace.uploadImgFromDisk(asRelativePath(path))

src/db/LiveDB.ts

@@ -1,11 +1,9 @@
-import { default as BetterSqlite3, default as SQL } from 'better-sqlite3'
-import { rmSync } from 'fs'
 import type { STATE } from '../state/state'
-
-import { makeAutoObservable } from 'mobx'
-import {
+import type { TableInfo } from 'src/db/TYPES_json'
+import type {
     ComfyPromptT,
     ComfySchemaT,
+    CustomDataT,
     DraftT,
     GraphT,
     Media3dDisplacementT,
@@ -17,10 +15,13 @@ import {
     RuntimeErrorT,
     StepT,
 } from 'src/db/TYPES.gen'
+
+import { rmSync } from 'fs'
+import { makeAutoObservable } from 'mobx'
 import { LiveTable } from './LiveTable'
+import { default as BetterSqlite3, default as SQL } from 'better-sqlite3'
 // models
 import { readFileSync } from 'fs'
-import { TableInfo } from 'src/db/TYPES_json'
 import { _applyAllMigrations } from 'src/db/_applyAllMigrations'
 import { _setupMigrationEngine } from 'src/db/_setupMigrationEngine'
 import { _listAllTables } from 'src/db/_listAllTables'
@@ -39,6 +40,7 @@ import { StepL } from '../models/Step'
 import { asRelativePath } from '../utils/fs/pathUtils'
 import { _printSchema } from 'src/db/_printSchema'
 import { MediaSplatL } from 'src/models/MediaSplat'
+import { CustomDataL } from 'src/models/CustomData'
 
 export type Indexed<T> = { [id: string]: T }
 
@@ -49,6 +51,7 @@ export class LiveDB {
 
     // tables ---------------------------------------------------------
     projects: LiveTable<ProjectT, ProjectL>
+    custom_datas: LiveTable<CustomDataT, CustomDataL>
     schemas: LiveTable<ComfySchemaT, SchemaL>
     comfy_prompts: LiveTable<ComfyPromptT, ComfyPromptL>
     media_texts: LiveTable<MediaTextT, MediaTextL>
@@ -87,6 +90,7 @@ export class LiveDB {
 
             // 3. create tables (after the store has benn made already observable)
             this.projects =              new LiveTable(this, 'project'              , '🤠', ProjectL, { singleton: true })
+            this.custom_datas =          new LiveTable(this, 'custom_data'          , '🎁', CustomDataL)
             this.schemas =               new LiveTable(this, 'comfy_schema'         , '📑', SchemaL, { singleton: true })
             this.comfy_prompts =         new LiveTable(this, 'comfy_prompt'         , '❓', ComfyPromptL)
             this.media_texts =           new LiveTable(this, 'media_text'           , '💬', MediaTextL)

src/db/LiveTable.ts

@@ -117,13 +117,16 @@ export class LiveTable<T extends BaseInstanceFields, L extends LiveInstance<T, L
             get tableName() { return this.table.name } // prettier-ignore
 
             update(changes: Partial<T>) {
+                console.log('💀 UPDATING ----------------------------')
                 // 0. check that changes is valid
                 if (Array.isArray(changes)) throw new Error('insert does not support arrays')
                 if (typeof changes !== 'object') throw new Error('insert does not support non-objects')
 
                 // 1. check if update is needed
-                const isSame = Object.keys(changes).every((k) => (this.data as any)[k] === (changes as any)[k])
-                if (isSame) return console.log('no need to update') // no need to update
+                const isSame = Object.keys(changes).every((k) => {
+                    return (this.data as any)[k] === (changes as any)[k]
+                })
+                if (isSame) return console.log('❌ no need to update') // no need to update
 
                 // 2. store the prev in case we have an onUpdate callback later
                 const prev = this.onUpdate //
@@ -154,14 +157,18 @@ export class LiveTable<T extends BaseInstanceFields, L extends LiveInstance<T, L
                             return [k, v]
                         }),
                     )
+
+                    // inject id and patch updatedAt
                     updatePayload.updatedAt = updatedAt
                     updatePayload.id = this.id
 
                     // update the data
-                    /*const data =*/ stmt.get(updatePayload) as any as T
+                    stmt.get(updatePayload) as any as T
 
+                    // assign the changes
+                    // 2023-12-02 rvion: for now, I'm not re-assigning from the returned values
                     Object.assign(this.data, changes)
-                    this.data.updatedAt = updatePayload
+                    this.data.updatedAt = updatedAt
                     this.onUpdate?.(prev, this.data)
                 } catch (e) {
                     console.log(updateSQL)

src/db/TYPES.d.ts

@@ -14,6 +14,7 @@ declare type TableNameInDB =
     | 'media_3d_displacement'
     | 'runtime_error'
     | 'media_splat'
+    | 'custom_data'
 
 declare type MigrationsID = Branded<string, { MigrationsID: true }>
 declare type UsersID = Branded<string, { UsersID: true }>
@@ -29,3 +30,4 @@ declare type MediaImageID = Branded<string, { MediaImageID: true }>
 declare type Media3dDisplacementID = Branded<string, { Media3dDisplacementID: true }>
 declare type RuntimeErrorID = Branded<string, { RuntimeErrorID: true }>
 declare type MediaSplatID = Branded<string, { MediaSplatID: true }>
+declare type CustomDataID = Branded<string, { CustomDataID: true }>

src/db/TYPES.gen.ts

@@ -622,6 +622,36 @@ export const MediaSplatFields = {
 }
 
 
+export const asCustomDataID = (s: string): CustomDataID => s as any
+export type CustomDataT = {
+    /** @default: "hex(randomblob(16))", sqlType: string */
+    id: CustomDataID;
+
+    /** @default: "now", sqlType: INTEGER */
+    createdAt: number;
+
+    /** @default: "now", sqlType: INTEGER */
+    updatedAt: number;
+
+    /** @default: "'{}'", sqlType: json */
+    json: T.CustomData_json;
+
+}
+export const CustomDataSchema = Type.Object({
+    id: Type.String(),
+    createdAt: Type.Number(),
+    updatedAt: Type.Number(),
+    json: T.CustomData_json_Schema,
+},{ additionalProperties: false })
+
+export const CustomDataFields = {
+    id: {cid:0,name:'id',type:'string',notnull:1,dflt_value:'hex(randomblob(16))',pk:1},
+    createdAt: {cid:1,name:'createdAt',type:'INTEGER',notnull:1,dflt_value:'now',pk:0},
+    updatedAt: {cid:2,name:'updatedAt',type:'INTEGER',notnull:1,dflt_value:'now',pk:0},
+    json: {cid:3,name:'json',type:'json',notnull:1,dflt_value:"'{}'",pk:0},
+}
+
+
 export const schemas = {
     migrations: new T.TableInfo(
         'migrations',
@@ -707,4 +737,10 @@ export const schemas = {
         MediaSplatFields,
         MediaSplatSchema,
     ),
+    custom_data: new T.TableInfo(
+        'custom_data',
+        'CustomData',
+        CustomDataFields,
+        CustomDataSchema,
+    ),
 }

src/db/TYPES_json.ts

@@ -15,6 +15,9 @@ export const Graph_comfyPromptJSON_Schema = Type.Record(Type.String(), Type.Any(
 export type Draft_appParams = Maybe<any>
 export const Draft_appParams_Schema = Type.Record(Type.String(), Type.Any())
 
+export type CustomData_json = any
+export const CustomData_json_Schema = Type.Any()
+
 export type Step_formResult = Maybe<any>
 export const Step_formResult_Schema = Type.Record(Type.String(), Type.Any())