fix: use only clownface for mappings to reduce memory

.changeset/fuzzy-moose-compare.md

@@ -0,0 +1,5 @@
+---
+"@cube-creator/core-api": patch
+---
+
+Reduce memory usage which would cause high memory spikes when saving large dimensions mappings (fixes #1444)

apis/core/lib/domain/dimension-mapping/DimensionMapping.ts

@@ -1,17 +1,18 @@
 import { Literal, NamedNode, Term } from 'rdf-js'
 import { Constructor, property } from '@tpluscode/rdfine'
 import { prov, rdf, schema } from '@tpluscode/rdf-ns-builders'
-import { Dictionary, KeyEntityPair } from '@rdfine/prov'
+import { Dictionary } from '@rdfine/prov'
 import { cc, md } from '@cube-creator/core/namespace'
 import TermMap from '@rdfjs/term-map'
 import $rdf from 'rdf-ext'
 import TermSet from '@rdfjs/term-set'
+import { GraphPointer } from 'clownface'
 
 interface DictionaryEx {
   about: NamedNode
   sharedDimensions: NamedNode[]
   onlyValidTerms: boolean
-  replaceEntries(entries: KeyEntityPair[]): { entriesChanged: boolean }
+  replaceEntries(dictionary: GraphPointer): { entriesChanged: boolean }
   changeSharedDimensions(sharedDimensions: NamedNode[]): void
   addMissingEntries(unmappedValues: Set<Literal>): void
   renameDimension(oldCube: NamedNode, newCube: NamedNode): void
@@ -38,33 +39,35 @@ export function ProvDictionaryMixinEx<Base extends Constructor<Dictionary>>(Reso
       this.sharedDimensions = sharedDimensions
     }
 
-    replaceEntries(entries: KeyEntityPair[]) {
-      const newEntries = new TermMap()
+    replaceEntries(dictionary: GraphPointer) {
+      let entriesAdded = false
 
-      const newEntryMap = entries.reduce<Map<Term, Term | undefined>>((map, { pairKey, pairEntity }) => {
+      const newEntries = dictionary.out(prov.hadDictionaryMember).toArray().reduce<Map<Term, Term | undefined>>((map, entryPtr) => {
+        const pairEntity = entryPtr.out(prov.pairEntity).term
         if (!pairEntity) {
           return map
         }
 
-        return pairKey ? map.set(pairKey, pairEntity?.id) : map
+        const pairKey = entryPtr.out(prov.pairKey).term
+        const currentEntry = this.pointer.out(prov.hadDictionaryMember).has(prov.pairKey, pairKey)
+        if (!currentEntry.term || !currentEntry.out(prov.pairEntity).term) {
+          entriesAdded = true
+        }
+        return pairKey ? map.set(pairKey, pairEntity) : map
       }, new TermMap())
 
       let entriesRemoved = false
       // Set values for current entries or remove
-      for (const entry of this.hadDictionaryMember) {
-        if (!entry.pairKey || !newEntryMap.has(entry.pairKey)) {
-          entriesRemoved = true
-          entry.pointer.deleteIn().deleteOut()
-          continue
-        }
+      for (const entryPtr of this.pointer.out(prov.hadDictionaryMember).toArray()) {
+        const pairKey = entryPtr.out(prov.pairKey).term
 
-        const newPairEntity = newEntryMap.get(entry.pairKey)
-        if (!entry.pairEntity && newPairEntity) {
-          newEntries.set(entry.pairKey, newPairEntity)
+        // mark as removed when not in the new map
+        if (pairKey && !newEntries.has(pairKey)) {
+          entriesRemoved = true
         }
 
-        entry.pairEntity = newPairEntity as any
-        newEntryMap.delete(entry.pairKey)
+        // remove form current graph
+        entryPtr.deleteIn().deleteOut()
       }
 
       this.pointer.any()
@@ -73,20 +76,17 @@ export function ProvDictionaryMixinEx<Base extends Constructor<Dictionary>>(Reso
         .forEach(entity => entity.deleteOut())
 
       // Insert new entries
-      this.hadDictionaryMember = [
-        ...this.hadDictionaryMember,
-        ...[...newEntryMap].reduce<KeyEntityPair[]>((arr, [pairKey, pairEntity]) => [...arr, {
-          pairKey,
-          pairEntity,
-        } as KeyEntityPair], []),
-      ]
-
-      for (const [pairKey, pairEntity] of newEntryMap) {
-        newEntries.set(pairKey, pairEntity)
+      for (const [key, entity] of newEntries) {
+        if (entity) {
+          this.pointer.addOut(prov.hadDictionaryMember, entry => {
+            entry.addOut(prov.pairKey, key)
+              .addOut(prov.pairEntity, entity)
+          })
+        }
       }
 
       return {
-        entriesChanged: newEntries.size > 0 || entriesRemoved,
+        entriesChanged: entriesAdded || entriesRemoved,
       }
     }
 

apis/core/lib/domain/dimension-mapping/update.ts

@@ -2,7 +2,9 @@ import { NamedNode } from 'rdf-js'
 import { GraphPointer } from 'clownface'
 import error from 'http-errors'
 import { Dictionary } from '@rdfine/prov'
-import { fromPointer } from '@rdfine/prov/lib/Dictionary'
+import { cc, md } from '@cube-creator/core/namespace'
+import { isNamedNode } from 'is-graph-pointer'
+import { schema } from '@tpluscode/rdf-ns-builders'
 import { ResourceStore } from '../../ResourceStore'
 
 interface UpdateDimensionMapping {
@@ -22,19 +24,18 @@ export async function update({
   store,
 }: UpdateDimensionMapping): Promise<Updated> {
   const dimensionMappings = await store.getResource<Dictionary>(resource)
-  const newMappings = fromPointer(mappings)
 
-  const sharedDimensions = newMappings.sharedDimensions
-  const dimension = newMappings.about
+  const sharedDimensions = mappings.out(cc.sharedDimension).filter(isNamedNode).terms
+  const dimension = mappings.out(schema.about).term!
 
   if (!dimension || !dimension.equals(dimensionMappings.about)) {
     throw new error.BadRequest('Unexpected value of schema:about')
   }
 
   dimensionMappings.changeSharedDimensions(sharedDimensions)
 
-  dimensionMappings.onlyValidTerms = newMappings.onlyValidTerms
-  const { entriesChanged } = dimensionMappings.replaceEntries(newMappings.hadDictionaryMember)
+  dimensionMappings.onlyValidTerms = mappings.out(md.onlyValidTerms).value === 'true'
+  const { entriesChanged } = dimensionMappings.replaceEntries(mappings)
 
   return {
     dimensionMapping: dimensionMappings.pointer,

apis/core/package.json

@@ -49,7 +49,7 @@
     "http-errors": "^2.0.0",
     "hydra-box": "^0.6.6",
     "hydra-box-middleware-shacl": "1.1.0",
-    "is-graph-pointer": "^1.2.0",
+    "is-graph-pointer": "^1.3.0",
     "is-stream": "^2",
     "jwks-rsa": "^3.0.0",
     "merge2": "^1.4.1",

apis/core/test/domain/dimension-mapping/DimensionMappings.test.ts

@@ -1,7 +1,6 @@
 import { NamedNode } from 'rdf-js'
 import { describe, it, beforeEach } from 'mocha'
 import { fromPointer } from '@rdfine/prov/lib/Dictionary'
-import { fromPointer as keyEntityPair } from '@rdfine/prov/lib/KeyEntityPair'
 import { GraphPointer } from 'clownface'
 import { expect } from 'chai'
 import $rdf from 'rdf-ext'
@@ -10,8 +9,6 @@ import TermSet from '@rdfjs/term-set'
 import { prov, xsd } from '@tpluscode/rdf-ns-builders'
 import { blankNode, namedNode } from '@cube-creator/testing/clownface'
 import '../../../lib/domain'
-import { Initializer } from '@tpluscode/rdfine/RdfResource'
-import { KeyEntityPair } from '@rdfine/prov'
 
 const wtd = namespace('http://www.wikidata.org/entity/')
 
@@ -186,16 +183,17 @@ describe('lib/domain/DimensionMappings', () => {
       })
 
       // when
-      const { entriesChanged } = dictionary.replaceEntries([
-        keyEntityPair(blankNode(), {
+      const newEntries = blankNode()
+      fromPointer(newEntries, {
+        hadDictionaryMember: [{
           pairKey: 'so2',
           pairEntity: wikidata.sulphurDioxide,
-        }),
-        keyEntityPair(blankNode(), {
+        }, {
           pairKey: 'co',
           pairEntity: wikidata.carbonMonoxide,
-        }),
-      ])
+        }],
+      })
+      const { entriesChanged } = dictionary.replaceEntries(newEntries)
 
       // then
       expect(entriesChanged).to.be.true
@@ -214,12 +212,14 @@ describe('lib/domain/DimensionMappings', () => {
       })
 
       // when
-      const { entriesChanged } = dictionary.replaceEntries([
-        keyEntityPair(blankNode(), {
+      const newEntries = blankNode()
+      fromPointer(newEntries, {
+        hadDictionaryMember: [{
           pairKey: 'co',
           pairEntity: wikidata.carbonMonoxide,
-        }),
-      ])
+        }],
+      })
+      const { entriesChanged } = dictionary.replaceEntries(newEntries)
 
       // then
       expect(entriesChanged).to.be.true
@@ -235,13 +235,14 @@ describe('lib/domain/DimensionMappings', () => {
       })
 
       // when
-      const so: Initializer<KeyEntityPair> = {
-        pairKey: 'co',
-        pairEntity: wikidata.carbonMonoxide,
-      }
-      const { entriesChanged } = dictionary.replaceEntries([
-        keyEntityPair(blankNode(), so),
-      ])
+      const newEntries = blankNode()
+      fromPointer(newEntries, {
+        hadDictionaryMember: [{
+          pairKey: 'co',
+          pairEntity: wikidata.carbonMonoxide,
+        }],
+      })
+      const { entriesChanged } = dictionary.replaceEntries(newEntries)
 
       // then
       expect(entriesChanged).to.be.false

apis/core/test/domain/dimension-mapping/update.test.ts

@@ -400,8 +400,8 @@ describe('domain/dimension-mapping/update', () => {
       expect(dimensionMapping.any().has([prov.pairKey, prov.pairEntity]).terms).to.have.length(2)
     })
 
-    it('keeps rdf:type prov:Entity triples of remaining pairs', () => {
-      expect(dimensionMapping.node(wikidata.carbonMonoxide).out(rdf.type).term).to.deep.eq(prov.Entity)
+    it('removes rdf:type prov:Entity triples of remaining pairs', () => {
+      expect(dimensionMapping.node(wikidata.carbonMonoxide).out(rdf.type).terms).to.be.empty
     })
   })
 

yarn.lock

@@ -9067,10 +9067,10 @@ is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1:
   dependencies:
     is-extglob "^2.1.1"
 
-is-graph-pointer@^1.2.0, is-graph-pointer@^1.2.2:
-  version "1.2.2"
-  resolved "https://registry.yarnpkg.com/is-graph-pointer/-/is-graph-pointer-1.2.2.tgz#1517f8f99f5347b2a02b92943df9d442771fdf6a"
-  integrity sha512-lF59hoMoZxh9T1bRrjQEiifS+FNJC0cw2rlKCOlDMCfrpwgDbHmhkKE1L3NfVQBF8eNgm6U40rdy0M9SzwCS2A==
+is-graph-pointer@^1.2.2, is-graph-pointer@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/is-graph-pointer/-/is-graph-pointer-1.3.0.tgz#e7b7bc04b5993c83b0cc3abe4e719a9bf6a5138f"
+  integrity sha512-EN+CsvlI55+QVtBd8Taqxi6KQmsQ0RRjkTi6TCoAYkJV1OvSqS5Gi2ypmjQ6a1hRERr19MV1NSS06BM6WGY4rg==
   dependencies:
     "@types/clownface" "^1.5.0"