UBERF-7934 Update activity from collaborator

models/activity/src/index.ts

@@ -124,7 +124,7 @@ export class TDocUpdateMessage extends TActivityMessage implements DocUpdateMess
 
   @Prop(TypeRef(core.class.TxCUD), core.string.Object)
   // @Index(IndexKind.Indexed)
-    txId!: Ref<TxCUD<Doc>>
+    txId?: Ref<TxCUD<Doc>>
 
   @Prop(TypeString(), core.string.Object)
   // @Index(IndexKind.Indexed)

plugins/activity/src/index.ts

@@ -93,7 +93,7 @@ export interface DocUpdateMessage extends ActivityMessage {
   objectId: Ref<Doc>
   objectClass: Ref<Class<Doc>>
 
-  txId: Ref<TxCUD<Doc>>
+  txId?: Ref<TxCUD<Doc>>
 
   action: DocUpdateAction
   updateCollection?: string

server-plugins/activity-resources/src/utils.ts

@@ -3,12 +3,9 @@ import {
   AttachedDoc,
   type Attribute,
   Class,
-  CollaborativeDoc,
-  collaborativeDocFromLastVersion,
   Collection,
   Doc,
   Hierarchy,
-  Markup,
   MeasureContext,
   Mixin,
   Ref,
@@ -18,18 +15,15 @@ import {
   TxCUD,
   TxMixin,
   TxProcessor,
-  TxUpdateDoc,
-  WorkspaceId
+  TxUpdateDoc
 } from '@hcengineering/core'
 import core from '@hcengineering/core/src/component'
 import { ActivityMessageControl, DocAttributeUpdates, DocUpdateAction } from '@hcengineering/activity'
 import { ActivityControl, DocObjectCache, getAllObjectTransactions } from '@hcengineering/server-activity'
 import { getDocCollaborators } from '@hcengineering/server-notification-resources'
 import notification from '@hcengineering/notification'
-import { StorageAdapter, TriggerControl } from '@hcengineering/server-core'
+import { TriggerControl } from '@hcengineering/server-core'
 import { translate } from '@hcengineering/platform'
-import { loadCollaborativeDoc } from '@hcengineering/collaboration'
-import { EmptyMarkup, yDocToMarkup } from '@hcengineering/text'
 
 function getAvailableAttributesKeys (tx: TxCUD<Doc>, hierarchy: Hierarchy): string[] {
   if (hierarchy.isDerived(tx._class, core.class.TxUpdateDoc)) {
@@ -301,11 +295,12 @@ export async function getTxAttributesUpdates (
       continue
     }
 
-    if (
-      hierarchy.isDerived(attrClass, core.class.TypeMarkup) ||
-      hierarchy.isDerived(attrClass, core.class.TypeCollaborativeDoc) ||
-      mixin === notification.mixin.Collaborators
-    ) {
+    if (attrClass === core.class.TypeCollaborativeDoc) {
+      // collaborative documents activity is handled by collaborator
+      continue
+    }
+
+    if (hierarchy.isDerived(attrClass, core.class.TypeMarkup) || mixin === notification.mixin.Collaborators) {
       if (docDiff === undefined) {
         docDiff = await getDocDiff(control, updateObject._class, updateObject._id, originTx._id, mixin, objectCache)
       }
@@ -331,16 +326,6 @@ export async function getTxAttributesUpdates (
       }
     }
 
-    // we don't want to show collaborative documents in activity
-    // instead we show their content as Markup
-    // TODO this should be generalized via activity extension
-    const attrType = mixin !== undefined ? hierarchy.findAttribute(mixin, key) : clazz
-    if (attrType?.type?._class === core.class.TypeCollaborativeDoc) {
-      attrClass = isMixin ? attrClass : core.class.TypeMarkup
-      attrValue = await getMarkup(ctx, control.storageAdapter, control.workspace, attrValue, key)
-      prevValue = await getMarkup(ctx, control.storageAdapter, control.workspace, prevValue, key)
-    }
-
     let setAttr = []
 
     if (Array.isArray(attrValue)) {
@@ -424,16 +409,3 @@ export function getCollectionAttribute (
 
   return undefined
 }
-
-async function getMarkup (
-  ctx: MeasureContext,
-  storage: StorageAdapter,
-  workspace: WorkspaceId,
-  value: CollaborativeDoc,
-  field: string
-): Promise<Markup> {
-  if (value === undefined) return EmptyMarkup
-  value = collaborativeDocFromLastVersion(value)
-  const ydoc = await loadCollaborativeDoc(storage, workspace, value, ctx)
-  return ydoc !== undefined ? yDocToMarkup(ydoc, field) : EmptyMarkup
-}

server/collaborator/package.json

@@ -46,6 +46,7 @@
     "@types/jest": "^29.5.5"
   },
   "dependencies": {
+    "@hcengineering/activity": "^0.6.0",
     "@hcengineering/analytics": "^0.6.0",
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/account": "^0.6.0",

server/collaborator/src/extensions/storage.ts

@@ -18,6 +18,7 @@ import { MeasureContext } from '@hcengineering/core'
 import {
   Document,
   Extension,
+  afterLoadDocumentPayload,
   afterUnloadDocumentPayload,
   onChangePayload,
   onConnectPayload,
@@ -28,15 +29,18 @@ import {
 import { Doc as YDoc } from 'yjs'
 import { Context, withContext } from '../context'
 import { CollabStorageAdapter } from '../storage/adapter'
+import { TransformerFactory } from '../types'
 
 export interface StorageConfiguration {
   ctx: MeasureContext
   adapter: CollabStorageAdapter
+  transformerFactory: TransformerFactory
 }
 
 export class StorageExtension implements Extension {
   private readonly configuration: StorageConfiguration
   private readonly collaborators = new Map<string, Set<string>>()
+  private readonly markups = new Map<string, Record<string, string>>()
 
   constructor (configuration: StorageConfiguration) {
     this.configuration = configuration
@@ -52,7 +56,15 @@ export class StorageExtension implements Extension {
     const { connectionId } = context
 
     this.configuration.ctx.info('load document', { documentName, connectionId })
-    return await this.loadDocument(documentName as DocumentId, context)
+    return await this.loadDocument(documentName, context)
+  }
+
+  async afterLoadDocument ({ context, documentName, document }: withContext<afterLoadDocumentPayload>): Promise<any> {
+    const { workspaceId } = context
+
+    // remember the markup for the document
+    const transformer = this.configuration.transformerFactory(workspaceId)
+    this.markups.set(documentName, transformer.fromYdoc(document))
   }
 
   async onStoreDocument ({ context, documentName, document }: withContext<onStoreDocumentPayload>): Promise<void> {
@@ -68,7 +80,7 @@ export class StorageExtension implements Extension {
     }
 
     this.collaborators.delete(documentName)
-    await this.storeDocument(documentName as DocumentId, document, context)
+    await this.storeDocument(documentName, document, context)
   }
 
   async onConnect ({ context, documentName, instance }: withContext<onConnectPayload>): Promise<any> {
@@ -91,36 +103,48 @@ export class StorageExtension implements Extension {
     }
 
     this.collaborators.delete(documentName)
-    await this.storeDocument(documentName as DocumentId, document, context)
+    await this.storeDocument(documentName, document, context)
   }
 
   async afterUnloadDocument ({ documentName }: afterUnloadDocumentPayload): Promise<any> {
     this.configuration.ctx.info('unload document', { documentName })
     this.collaborators.delete(documentName)
+    this.markups.delete(documentName)
   }
 
-  async loadDocument (documentId: DocumentId, context: Context): Promise<YDoc | undefined> {
+  private async loadDocument (documentName: string, context: Context): Promise<YDoc | undefined> {
     const { ctx, adapter } = this.configuration
 
     try {
       return await ctx.with('load-document', {}, async (ctx) => {
-        return await adapter.loadDocument(ctx, documentId, context)
+        return await adapter.loadDocument(ctx, documentName as DocumentId, context)
       })
     } catch (err) {
-      ctx.error('failed to load document', { documentId, error: err })
+      ctx.error('failed to load document', { documentName, error: err })
       throw new Error('Failed to load document')
     }
   }
 
-  async storeDocument (documentId: DocumentId, document: Document, context: Context): Promise<void> {
+  private async storeDocument (documentName: string, document: Document, context: Context): Promise<void> {
     const { ctx, adapter } = this.configuration
+    const { workspaceId } = context
 
     try {
+      const transformer = this.configuration.transformerFactory(workspaceId)
+
+      const prevMarkup = this.markups.get(documentName) ?? {}
+      const currMarkup = transformer.fromYdoc(document)
+
       await ctx.with('save-document', {}, async (ctx) => {
-        await adapter.saveDocument(ctx, documentId, document, context)
+        await adapter.saveDocument(ctx, documentName as DocumentId, document, context, {
+          prev: prevMarkup,
+          curr: currMarkup
+        })
       })
+
+      this.markups.set(documentName, currMarkup)
     } catch (err) {
-      ctx.error('failed to save document', { documentId, error: err })
+      ctx.error('failed to save document', { documentName, error: err })
       throw new Error('Failed to save document')
     }
   }

server/collaborator/src/server.ts

@@ -109,7 +109,8 @@ export async function start (ctx: MeasureContext, config: Config, storageAdapter
       }),
       new StorageExtension({
         ctx: extensionsCtx.newChild('storage', {}),
-        adapter: new PlatformStorageAdapter(storageAdapter)
+        adapter: new PlatformStorageAdapter(storageAdapter),
+        transformerFactory
       })
     ]
   })

server/collaborator/src/storage/adapter.ts

@@ -20,5 +20,14 @@ import { Context } from '../context'
 
 export interface CollabStorageAdapter {
   loadDocument: (ctx: MeasureContext, documentId: DocumentId, context: Context) => Promise<YDoc | undefined>
-  saveDocument: (ctx: MeasureContext, documentId: DocumentId, document: YDoc, context: Context) => Promise<void>
+  saveDocument: (
+    ctx: MeasureContext,
+    documentId: DocumentId,
+    document: YDoc,
+    context: Context,
+    markup: {
+      prev: Record<string, string>
+      curr: Record<string, string>
+    }
+  ) => Promise<void>
 }

server/collaborator/src/storage/platform.ts

@@ -13,6 +13,7 @@
 // limitations under the License.
 //
 
+import activity, { DocUpdateMessage } from '@hcengineering/activity'
 import {
   YDocVersion,
   loadCollaborativeDoc,
@@ -26,13 +27,15 @@ import {
   parsePlatformDocumentId
 } from '@hcengineering/collaborator-client'
 import core, {
+  AttachedData,
   CollaborativeDoc,
   MeasureContext,
   TxOperations,
   collaborativeDocWithLastVersion
 } from '@hcengineering/core'
 import { StorageAdapter } from '@hcengineering/server-core'
 import { Doc as YDoc } from 'yjs'
+
 import { Context } from '../context'
 
 import { CollabStorageAdapter } from './adapter'
@@ -78,7 +81,16 @@ export class PlatformStorageAdapter implements CollabStorageAdapter {
     return undefined
   }
 
-  async saveDocument (ctx: MeasureContext, documentId: DocumentId, document: YDoc, context: Context): Promise<void> {
+  async saveDocument (
+    ctx: MeasureContext,
+    documentId: DocumentId,
+    document: YDoc,
+    context: Context,
+    markup: {
+      prev: Record<string, string>
+      curr: Record<string, string>
+    }
+  ): Promise<void> {
     const { clientFactory } = context
 
     const client = await ctx.with('connect', {}, async () => {
@@ -108,7 +120,7 @@ export class PlatformStorageAdapter implements CollabStorageAdapter {
       if (platformDocumentId !== undefined) {
         ctx.info('save document content to platform', { documentId, platformDocumentId })
         await ctx.with('save-to-platform', {}, async (ctx) => {
-          await this.saveDocumentToPlatform(ctx, client, documentId, platformDocumentId, snapshot)
+          await this.saveDocumentToPlatform(ctx, client, documentId, platformDocumentId, snapshot, markup)
         })
       }
     } finally {
@@ -177,7 +189,11 @@ export class PlatformStorageAdapter implements CollabStorageAdapter {
     client: Omit<TxOperations, 'close'>,
     documentName: string,
     platformDocumentId: PlatformDocumentId,
-    snapshot: YDocVersion | undefined
+    snapshot: YDocVersion | undefined,
+    markup: {
+      prev: Record<string, string>
+      curr: Record<string, string>
+    }
   ): Promise<void> {
     const { objectClass, objectId, objectAttr } = parsePlatformDocumentId(platformDocumentId)
 
@@ -197,19 +213,43 @@ export class PlatformStorageAdapter implements CollabStorageAdapter {
     }
 
     const hierarchy = client.getHierarchy()
-    if (hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc)) {
-      const collaborativeDoc = (current as any)[objectAttr] as CollaborativeDoc
-      const newCollaborativeDoc =
-        snapshot !== undefined
-          ? collaborativeDocWithLastVersion(collaborativeDoc, snapshot.versionId)
-          : collaborativeDoc
-
-      await ctx.with('update', {}, async () => {
-        await client.diffUpdate(current, { [objectAttr]: newCollaborativeDoc })
-      })
-    } else {
-      ctx.error('unsupported attribute type', { documentName, objectClass, objectAttr })
+    if (!hierarchy.isDerived(attribute.type._class, core.class.TypeCollaborativeDoc)) {
+      ctx.warn('unsupported attribute type', { documentName, objectClass, objectAttr })
+      return
     }
+
+    const collaborativeDoc = (current as any)[objectAttr] as CollaborativeDoc
+    const newCollaborativeDoc =
+      snapshot !== undefined ? collaborativeDocWithLastVersion(collaborativeDoc, snapshot.versionId) : collaborativeDoc
+
+    await ctx.with('update', {}, async () => {
+      await client.diffUpdate(current, { [objectAttr]: newCollaborativeDoc })
+    })
+
+    await ctx.with('activity', {}, async () => {
+      const data: AttachedData<DocUpdateMessage> = {
+        objectId,
+        objectClass,
+        action: 'update',
+        attributeUpdates: {
+          attrKey: objectAttr,
+          attrClass: core.class.TypeMarkup,
+          prevValue: markup.prev[objectAttr],
+          set: [markup.curr[objectAttr]],
+          added: [],
+          removed: [],
+          isMixin: hierarchy.isMixin(objectClass)
+        }
+      }
+      await client.addCollection(
+        activity.class.DocUpdateMessage,
+        current.space,
+        current._id,
+        current._class,
+        'docUpdateMessages',
+        data
+      )
+    })
   }
 }
 

server/collaborator/src/transformers/markup.ts

@@ -27,7 +27,20 @@ export class MarkupTransformer implements Transformer {
 
   fromYdoc (document: Doc, fieldName?: string | string[] | undefined): any {
     const json = this.transformer.fromYdoc(document, fieldName)
-    return jsonToMarkup(json)
+    if (typeof fieldName === 'string') {
+      return jsonToMarkup(json)
+    }
+
+    if (fieldName === undefined || fieldName.length === 0) {
+      fieldName = Array.from(document.share.keys())
+    }
+
+    const data: Record<string, string> = {}
+    fieldName?.forEach((field) => {
+      data[field] = jsonToMarkup(json[field])
+    })
+
+    return data
   }
 
   toYdoc (document: any, fieldName: string): Doc {