diff --git a/.changeset/fuzzy-moose-compare.md b/.changeset/fuzzy-moose-compare.md new file mode 100644 index 000000000..0d2d85c2e --- /dev/null +++ b/.changeset/fuzzy-moose-compare.md @@ -0,0 +1,5 @@ +--- +"@cube-creator/core-api": patch +--- + +Reduce memory usage which would cause high memory spikes when saving large dimensions mappings (fixes #1444) diff --git a/apis/core/lib/domain/dimension-mapping/DimensionMapping.ts b/apis/core/lib/domain/dimension-mapping/DimensionMapping.ts index 0e7687343..4bba05332 100644 --- a/apis/core/lib/domain/dimension-mapping/DimensionMapping.ts +++ b/apis/core/lib/domain/dimension-mapping/DimensionMapping.ts @@ -1,17 +1,18 @@ import { Literal, NamedNode, Term } from 'rdf-js' import { Constructor, property } from '@tpluscode/rdfine' import { prov, rdf, schema } from '@tpluscode/rdf-ns-builders' -import { Dictionary, KeyEntityPair } from '@rdfine/prov' +import { Dictionary } from '@rdfine/prov' import { cc, md } from '@cube-creator/core/namespace' import TermMap from '@rdfjs/term-map' import $rdf from 'rdf-ext' import TermSet from '@rdfjs/term-set' +import { GraphPointer } from 'clownface' interface DictionaryEx { about: NamedNode sharedDimensions: NamedNode[] onlyValidTerms: boolean - replaceEntries(entries: KeyEntityPair[]): { entriesChanged: boolean } + replaceEntries(dictionary: GraphPointer): { entriesChanged: boolean } changeSharedDimensions(sharedDimensions: NamedNode[]): void addMissingEntries(unmappedValues: Set): void renameDimension(oldCube: NamedNode, newCube: NamedNode): void @@ -38,33 +39,35 @@ export function ProvDictionaryMixinEx>(Reso this.sharedDimensions = sharedDimensions } - replaceEntries(entries: KeyEntityPair[]) { - const newEntries = new TermMap() + replaceEntries(dictionary: GraphPointer) { + let entriesAdded = false - const newEntryMap = entries.reduce>((map, { pairKey, pairEntity }) => { + const newEntries = dictionary.out(prov.hadDictionaryMember).toArray().reduce>((map, entryPtr) => { + const pairEntity = entryPtr.out(prov.pairEntity).term if (!pairEntity) { return map } - return pairKey ? map.set(pairKey, pairEntity?.id) : map + const pairKey = entryPtr.out(prov.pairKey).term + const currentEntry = this.pointer.out(prov.hadDictionaryMember).has(prov.pairKey, pairKey) + if (!currentEntry.term || !currentEntry.out(prov.pairEntity).term) { + entriesAdded = true + } + return pairKey ? map.set(pairKey, pairEntity) : map }, new TermMap()) let entriesRemoved = false // Set values for current entries or remove - for (const entry of this.hadDictionaryMember) { - if (!entry.pairKey || !newEntryMap.has(entry.pairKey)) { - entriesRemoved = true - entry.pointer.deleteIn().deleteOut() - continue - } + for (const entryPtr of this.pointer.out(prov.hadDictionaryMember).toArray()) { + const pairKey = entryPtr.out(prov.pairKey).term - const newPairEntity = newEntryMap.get(entry.pairKey) - if (!entry.pairEntity && newPairEntity) { - newEntries.set(entry.pairKey, newPairEntity) + // mark as removed when not in the new map + if (pairKey && !newEntries.has(pairKey)) { + entriesRemoved = true } - entry.pairEntity = newPairEntity as any - newEntryMap.delete(entry.pairKey) + // remove form current graph + entryPtr.deleteIn().deleteOut() } this.pointer.any() @@ -73,20 +76,17 @@ export function ProvDictionaryMixinEx>(Reso .forEach(entity => entity.deleteOut()) // Insert new entries - this.hadDictionaryMember = [ - ...this.hadDictionaryMember, - ...[...newEntryMap].reduce((arr, [pairKey, pairEntity]) => [...arr, { - pairKey, - pairEntity, - } as KeyEntityPair], []), - ] - - for (const [pairKey, pairEntity] of newEntryMap) { - newEntries.set(pairKey, pairEntity) + for (const [key, entity] of newEntries) { + if (entity) { + this.pointer.addOut(prov.hadDictionaryMember, entry => { + entry.addOut(prov.pairKey, key) + .addOut(prov.pairEntity, entity) + }) + } } return { - entriesChanged: newEntries.size > 0 || entriesRemoved, + entriesChanged: entriesAdded || entriesRemoved, } } diff --git a/apis/core/lib/domain/dimension-mapping/update.ts b/apis/core/lib/domain/dimension-mapping/update.ts index c93ce3954..1583596d4 100644 --- a/apis/core/lib/domain/dimension-mapping/update.ts +++ b/apis/core/lib/domain/dimension-mapping/update.ts @@ -2,7 +2,9 @@ import { NamedNode } from 'rdf-js' import { GraphPointer } from 'clownface' import error from 'http-errors' import { Dictionary } from '@rdfine/prov' -import { fromPointer } from '@rdfine/prov/lib/Dictionary' +import { cc, md } from '@cube-creator/core/namespace' +import { isNamedNode } from 'is-graph-pointer' +import { schema } from '@tpluscode/rdf-ns-builders' import { ResourceStore } from '../../ResourceStore' interface UpdateDimensionMapping { @@ -22,10 +24,9 @@ export async function update({ store, }: UpdateDimensionMapping): Promise { const dimensionMappings = await store.getResource(resource) - const newMappings = fromPointer(mappings) - const sharedDimensions = newMappings.sharedDimensions - const dimension = newMappings.about + const sharedDimensions = mappings.out(cc.sharedDimension).filter(isNamedNode).terms + const dimension = mappings.out(schema.about).term! if (!dimension || !dimension.equals(dimensionMappings.about)) { throw new error.BadRequest('Unexpected value of schema:about') @@ -33,8 +34,8 @@ export async function update({ dimensionMappings.changeSharedDimensions(sharedDimensions) - dimensionMappings.onlyValidTerms = newMappings.onlyValidTerms - const { entriesChanged } = dimensionMappings.replaceEntries(newMappings.hadDictionaryMember) + dimensionMappings.onlyValidTerms = mappings.out(md.onlyValidTerms).value === 'true' + const { entriesChanged } = dimensionMappings.replaceEntries(mappings) return { dimensionMapping: dimensionMappings.pointer, diff --git a/apis/core/package.json b/apis/core/package.json index 56e697db7..49b207548 100644 --- a/apis/core/package.json +++ b/apis/core/package.json @@ -49,7 +49,7 @@ "http-errors": "^2.0.0", "hydra-box": "^0.6.6", "hydra-box-middleware-shacl": "1.1.0", - "is-graph-pointer": "^1.2.0", + "is-graph-pointer": "^1.3.0", "is-stream": "^2", "jwks-rsa": "^3.0.0", "merge2": "^1.4.1", diff --git a/apis/core/test/domain/dimension-mapping/DimensionMappings.test.ts b/apis/core/test/domain/dimension-mapping/DimensionMappings.test.ts index 9d3b516e4..edd6d30c7 100644 --- a/apis/core/test/domain/dimension-mapping/DimensionMappings.test.ts +++ b/apis/core/test/domain/dimension-mapping/DimensionMappings.test.ts @@ -1,7 +1,6 @@ import { NamedNode } from 'rdf-js' import { describe, it, beforeEach } from 'mocha' import { fromPointer } from '@rdfine/prov/lib/Dictionary' -import { fromPointer as keyEntityPair } from '@rdfine/prov/lib/KeyEntityPair' import { GraphPointer } from 'clownface' import { expect } from 'chai' import $rdf from 'rdf-ext' @@ -10,8 +9,6 @@ import TermSet from '@rdfjs/term-set' import { prov, xsd } from '@tpluscode/rdf-ns-builders' import { blankNode, namedNode } from '@cube-creator/testing/clownface' import '../../../lib/domain' -import { Initializer } from '@tpluscode/rdfine/RdfResource' -import { KeyEntityPair } from '@rdfine/prov' const wtd = namespace('http://www.wikidata.org/entity/') @@ -186,16 +183,17 @@ describe('lib/domain/DimensionMappings', () => { }) // when - const { entriesChanged } = dictionary.replaceEntries([ - keyEntityPair(blankNode(), { + const newEntries = blankNode() + fromPointer(newEntries, { + hadDictionaryMember: [{ pairKey: 'so2', pairEntity: wikidata.sulphurDioxide, - }), - keyEntityPair(blankNode(), { + }, { pairKey: 'co', pairEntity: wikidata.carbonMonoxide, - }), - ]) + }], + }) + const { entriesChanged } = dictionary.replaceEntries(newEntries) // then expect(entriesChanged).to.be.true @@ -214,12 +212,14 @@ describe('lib/domain/DimensionMappings', () => { }) // when - const { entriesChanged } = dictionary.replaceEntries([ - keyEntityPair(blankNode(), { + const newEntries = blankNode() + fromPointer(newEntries, { + hadDictionaryMember: [{ pairKey: 'co', pairEntity: wikidata.carbonMonoxide, - }), - ]) + }], + }) + const { entriesChanged } = dictionary.replaceEntries(newEntries) // then expect(entriesChanged).to.be.true @@ -235,13 +235,14 @@ describe('lib/domain/DimensionMappings', () => { }) // when - const so: Initializer = { - pairKey: 'co', - pairEntity: wikidata.carbonMonoxide, - } - const { entriesChanged } = dictionary.replaceEntries([ - keyEntityPair(blankNode(), so), - ]) + const newEntries = blankNode() + fromPointer(newEntries, { + hadDictionaryMember: [{ + pairKey: 'co', + pairEntity: wikidata.carbonMonoxide, + }], + }) + const { entriesChanged } = dictionary.replaceEntries(newEntries) // then expect(entriesChanged).to.be.false diff --git a/apis/core/test/domain/dimension-mapping/update.test.ts b/apis/core/test/domain/dimension-mapping/update.test.ts index 0ff161fe9..de50d25ac 100644 --- a/apis/core/test/domain/dimension-mapping/update.test.ts +++ b/apis/core/test/domain/dimension-mapping/update.test.ts @@ -400,8 +400,8 @@ describe('domain/dimension-mapping/update', () => { expect(dimensionMapping.any().has([prov.pairKey, prov.pairEntity]).terms).to.have.length(2) }) - it('keeps rdf:type prov:Entity triples of remaining pairs', () => { - expect(dimensionMapping.node(wikidata.carbonMonoxide).out(rdf.type).term).to.deep.eq(prov.Entity) + it('removes rdf:type prov:Entity triples of remaining pairs', () => { + expect(dimensionMapping.node(wikidata.carbonMonoxide).out(rdf.type).terms).to.be.empty }) }) diff --git a/yarn.lock b/yarn.lock index 29a5398a9..c72d06154 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9067,10 +9067,10 @@ is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1: dependencies: is-extglob "^2.1.1" -is-graph-pointer@^1.2.0, is-graph-pointer@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/is-graph-pointer/-/is-graph-pointer-1.2.2.tgz#1517f8f99f5347b2a02b92943df9d442771fdf6a" - integrity sha512-lF59hoMoZxh9T1bRrjQEiifS+FNJC0cw2rlKCOlDMCfrpwgDbHmhkKE1L3NfVQBF8eNgm6U40rdy0M9SzwCS2A== +is-graph-pointer@^1.2.2, is-graph-pointer@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/is-graph-pointer/-/is-graph-pointer-1.3.0.tgz#e7b7bc04b5993c83b0cc3abe4e719a9bf6a5138f" + integrity sha512-EN+CsvlI55+QVtBd8Taqxi6KQmsQ0RRjkTi6TCoAYkJV1OvSqS5Gi2ypmjQ6a1hRERr19MV1NSS06BM6WGY4rg== dependencies: "@types/clownface" "^1.5.0"