fix: invalid view query during publishing

zazuko · Oct 5, 2023 · 5a2e353 · 5a2e353
1 parent ba773c7
commit 5a2e353
Show file tree

Hide file tree

Showing 15 changed files with 1,029 additions and 540 deletions.
diff --git a/.changeset/small-falcons-peel.md b/.changeset/small-falcons-peel.md
@@ -0,0 +1,5 @@
+---
+"@view-builder/api": patch
+---
+
+When publishing views, there was an invalid query being generated
diff --git a/apps/api/lib/errors.js b/apps/api/lib/errors.js
@@ -1,6 +1,6 @@
 /* eslint-disable class-methods-use-this */
 import { ProblemDocument } from 'http-problem-details'
-import { fromPointer } from '@rdfine/shacl/lib/ValidationReport'
+import rdf from '@view-builder/core/env.js'
 import { hex } from '@hydrofoil/vocabularies/builders'
 
 export class ValidationErrorMapper {
@@ -9,7 +9,7 @@ export class ValidationErrorMapper {
   }
 
   mapError(error) {
-    const report = error.reports.map(ptr => fromPointer(ptr).toJSON())
+    const report = error.reports.map(ptr => rdf.rdfine.sh.ValidationReport(ptr).toJSON())
 
     const problem = new ProblemDocument({
       title: 'Data issues were found',

diff --git a/apps/api/package.json b/apps/api/package.json
@@ -8,24 +8,23 @@
   },
   "dependencies": {
     "@hydrofoil/knossos": "^0.9.11",
-    "@hydrofoil/vocabularies": "^0.3.3",
-    "@rdfine/hydra": "^0.8.6",
-    "@rdfine/shacl": "^0.8.7",
-    "@tpluscode/rdf-ns-builders": "^2.0.1",
-    "@tpluscode/rdfine": "^0.5.41",
-    "@tpluscode/sparql-builder": "^0.3.23",
+    "@hydrofoil/vocabularies": "^1.1.1",
+    "@rdfine/hydra": "^0.10.2",
+    "@rdfine/shacl": "^0.10.2",
+    "@tpluscode/rdf-ns-builders": "^4",
+    "@tpluscode/rdfine": "^0.7.4",
+    "@tpluscode/sparql-builder": "^1",
     "@view-builder/core": "^0.0.0",
     "@view-builder/publish-views": "^0.0.4",
-    "@zazuko/vocabulary-extras": "^1.2.3",
-    "clownface": "^1.5.1",
+    "@zazuko/vocabulary-extras": "^2.0.1",
+    "clownface": "^2.0.1",
     "cors": "^2.8.5",
     "express": "^4.18.1",
     "express-basic-auth": "^1.2.1",
     "http-problem-details": "^0.1.5",
-    "is-graph-pointer": "^1.2.2",
+    "is-graph-pointer": "^2",
     "middleware-async": "^1.3.5",
     "rdf-dataset-ext": "^1.0.1",
-    "rdf-ext": "^2.0.1",
     "rdf-literal": "^1.3.0",
     "sparql-http-client": "^2.4.1",
     "tempy": "^3.0.0"

diff --git a/apps/api/resources.dev/view/640.ttl b/apps/api/resources.dev/view/640.ttl
@@ -0,0 +1,125 @@
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix schema: <http://schema.org/> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+
+<> a
+    <https://cube.link/view/View>,
+    </api/View> ;
+  schema:sameAs <https://ld.stadt-zuerich.ch/statistics/view/BEV353OD3531> ;
+  schema:author <https://view-builder.ld.integ.stadt-zuerich.ch/user/sszmop> ;
+  <https://cube.link/view/dimension>
+    [
+      rdf:type <https://cube.link/view/Dimension> ;
+      <https://cube.link/view/from>
+        [
+          <https://cube.link/view/source>
+            <#source-GSAFVdiQxRnKyjdbADwAT> ;
+          <https://cube.link/view/path> <https://ld.stadt-zuerich.ch/statistics/measure/ZUZ> ;
+        ] ;
+      rdfs:label
+        "Measure Zuzüge von Personen (wirtschaftlich) (Zuzüge von Personen (wirtschaftlich) nach Alter (20-Jahres-Altersklasse80), Herkunft und Geschlecht, gemessen nach Ereignisdatum)" ;
+      <https://view-builder.described.at/generated> true ;
+    ],
+    _:genid-ca5f09348634420db5e962546edc79a2-3B61177AC6B4C6D3F94321A240DDAFDA,
+    _:genid-ca5f09348634420db5e962546edc79a2-7C780A7C073A806916F63CD4B645AF36,
+    [
+      rdf:type <https://cube.link/view/Dimension> ;
+      <https://cube.link/view/from>
+        [
+          <https://cube.link/view/source>
+            <#source-GSAFVdiQxRnKyjdbADwAT> ;
+          <https://cube.link/view/path> <https://ld.stadt-zuerich.ch/statistics/property/ALT> ;
+        ] ;
+      rdfs:label "Key Alter" ;
+      <https://view-builder.described.at/generated> true ;
+    ],
+    [
+      rdf:type <https://cube.link/view/Dimension> ;
+      <https://cube.link/view/from>
+        [
+          <https://cube.link/view/source>
+            <#source-GSAFVdiQxRnKyjdbADwAT> ;
+          <https://cube.link/view/path> <https://ld.stadt-zuerich.ch/statistics/property/SEX> ;
+        ] ;
+      rdfs:label "Key Geschlecht" ;
+      <https://view-builder.described.at/generated> true ;
+    ],
+    [
+      rdf:type <https://cube.link/view/Dimension> ;
+      <https://cube.link/view/from>
+        [
+          <https://cube.link/view/source>
+            <#source-GSAFVdiQxRnKyjdbADwAT> ;
+          <https://cube.link/view/path> <https://ld.stadt-zuerich.ch/statistics/property/HEL> ;
+        ] ;
+      rdfs:label "Key Heimatland" ;
+      <https://view-builder.described.at/generated> true ;
+    ],
+    [
+      rdf:type <https://cube.link/view/Dimension> ;
+      <https://cube.link/view/from>
+        [
+          <https://cube.link/view/source>
+            <#source-GSAFVdiQxRnKyjdbADwAT> ;
+          <https://cube.link/view/path> <https://ld.stadt-zuerich.ch/statistics/property/ZEK> ;
+        ] ;
+      rdfs:label "Key Zeitkonzept" ;
+      <https://view-builder.described.at/generated> true ;
+    ] ;
+  <https://view-builder.described.at/publish> true ;
+  schema:name "Zuzüge von auswärts nach Tag, Stadtquartier, Geschlecht, Altersgruppe und Herkunft" ;
+  schema:alternateName "BEV353OD3531" ;
+  <https://cube.link/view/projection>
+    [
+      <https://cube.link/view/limit> 100 ;
+    ] ;
+  <https://view-builder.described.at/source>
+    <#source-GSAFVdiQxRnKyjdbADwAT> ;
+  <https://cube.link/view/filter>
+    [
+      rdf:type <https://cube.link/view/Filter> ;
+      rdfs:label "Raum is one of \"QuartiereZH\"" ;
+      <https://view-builder.described.at/baseDimension>
+        _:genid-ca5f09348634420db5e962546edc79a2-7C780A7C073A806916F63CD4B645AF36 ;
+      <https://view-builder.described.at/filterTermSet> <https://ld.stadt-zuerich.ch/statistics/termset/QuartiereZH> ;
+      <https://view-builder.described.at/drillDownProperty> schema:name ;
+    ],
+    [
+      rdf:type <https://cube.link/view/Filter> ;
+      rdfs:label "Zeit is one of \"Tag\"" ;
+      <https://view-builder.described.at/baseDimension>
+        _:genid-ca5f09348634420db5e962546edc79a2-3B61177AC6B4C6D3F94321A240DDAFDA ;
+      <https://view-builder.described.at/filterTermSet> <https://ld.stadt-zuerich.ch/statistics/termset/Tag> ;
+      <https://view-builder.described.at/drillDownProperty> schema:name ;
+    ],
+    [
+      rdf:type <https://cube.link/view/Filter> ;
+      rdfs:label "Zeit is one of \"Periode\"" ;
+      <https://view-builder.described.at/baseDimension>
+        _:genid-ca5f09348634420db5e962546edc79a2-3B61177AC6B4C6D3F94321A240DDAFDA ;
+      <https://view-builder.described.at/filterTermSet> <https://ld.stadt-zuerich.ch/statistics/termset/Periode> ;
+      <https://view-builder.described.at/drillDownProperty> schema:name ;
+    ] ;
+  schema:isBasedOn <https://ld.stadt-zuerich.ch/statistics/meta/object/640> ;
+  <https://ld.stadt-zuerich.ch/schema/metadataCreator> <https://view-builder.ld.integ.stadt-zuerich.ch/user/sszmop> .
+
+_:genid-ca5f09348634420db5e962546edc79a2-3B61177AC6B4C6D3F94321A240DDAFDA a <https://cube.link/view/Dimension> ;
+  <https://cube.link/view/from>
+    [
+      <https://cube.link/view/source>
+        <#source-GSAFVdiQxRnKyjdbADwAT> ;
+      <https://cube.link/view/path> <https://ld.stadt-zuerich.ch/statistics/property/ZEIT> ;
+    ] ;
+  rdfs:label "Key Zeit" ;
+  <https://view-builder.described.at/generated> true .
+
+_:genid-ca5f09348634420db5e962546edc79a2-7C780A7C073A806916F63CD4B645AF36 a <https://cube.link/view/Dimension> ;
+  <https://cube.link/view/from>
+    [
+      <https://cube.link/view/source>
+        <#source-GSAFVdiQxRnKyjdbADwAT> ;
+      <https://cube.link/view/path> <https://ld.stadt-zuerich.ch/statistics/property/RAUM> ;
+    ] ;
+  rdfs:label "Key Raum" ;
+  <https://view-builder.described.at/generated> true .
+
diff --git a/apps/www/package.json b/apps/www/package.json
@@ -19,7 +19,7 @@
     "@hydrofoil/vocabularies": "^0.3.2",
     "@material/mwc-drawer": "^0.27",
     "@ngard/tiny-difference": "^1.0.0",
-    "@rdfine/hydra": "^0.8.6",
+    "@rdfine/hydra": "^0.10.2",
     "@rdfjs-elements/lit-helpers": "^0.3.4",
     "@rdfjs-elements/formats-pretty": "^0.5.1",
     "@rdfjs/data-model": "^2.0.1",
@@ -28,20 +28,20 @@
     "@rdfjs/term-set": "^2.0.0",
     "@rdfjs/traverser": "^0.1.0",
     "@shoelace-style/shoelace": "^2.0.0-beta.80",
-    "@tpluscode/rdf-ns-builders": "^2.0.1",
-    "@tpluscode/rdf-string": "^0.2.26",
-    "@tpluscode/sparql-builder": "^0.3.23",
+    "@tpluscode/rdf-ns-builders": "^4",
+    "@tpluscode/rdf-string": "^1.0.3",
+    "@tpluscode/sparql-builder": "^1.1.0",
     "content-disposition": "^0.5.4",
     "@view-builder/core": "0.0.0",
     "@view-builder/view-util": "0.0.1",
-    "@zazuko/rdf-vocabularies": "^2022.6.29",
-    "@zazuko/vocabulary-extras": "^1.2.3",
+    "@zazuko/prefixes": "^2.0.0",
+    "@zazuko/vocabulary-extras": "^2.0.1",
     "alcaeus": "^2.3.2",
-    "clownface": "^1.5.1",
-    "clownface-shacl-path": "^1.3.2",
+    "clownface": "^2",
+    "clownface-shacl-path": "^2.0.2",
     "get-stream": "^6.0.1",
     "is-absolute-url": "^4.0.1",
-    "is-graph-pointer": "^1.2.2",
+    "is-graph-pointer": "^2.1.0",
     "lit": "^2.3.0",
     "n3": "^1.16.2",
     "nanoid": "^4.0.0",
@@ -58,7 +58,7 @@
     "@babel/plugin-proposal-optional-chaining": "^7.16.5",
     "@babel/preset-typescript": "^7.16.5",
     "@open-wc/building-webpack": "^2.13.46",
-    "@rdfine/shacl": "^0.8.7",
+    "@rdfine/shacl": "^0.10.2",
     "@view-builder/testing": "0.0.0",
     "chai": "^4.3.6",
     "copy-webpack-plugin": "^6",

diff --git a/apps/www/src/element/ssz-shacl-button.js b/apps/www/src/element/ssz-shacl-button.js
@@ -1,6 +1,6 @@
 import { css, html, LitElement } from 'lit'
 import '@shoelace-style/shoelace/dist/components/icon-button/icon-button.js'
-import prefixes from '@zazuko/rdf-vocabularies/prefixes'
+import prefixes from '@zazuko/prefixes'
 import prefixesExtras from '@zazuko/vocabulary-extras/prefixes'
 import { view } from '@view-builder/core/ns.js'
 

diff --git a/package.json b/package.json
@@ -24,7 +24,7 @@
   "author": "Zazuko GmbH <[email protected]>",
   "repository": "https://github.com/zazuko/ssz-view-builder",
   "dependencies": {
-    "@hydrofoil/knossos": "^0.9.6",
+    "@hydrofoil/knossos": "^0.9.11",
     "@hydrofoil/talos": "^0.6.2",
     "compression": "^1.7.4",
     "express": "^4.17.1",
@@ -40,12 +40,12 @@
     "url-state": "3.0.2"
   },
   "devDependencies": {
-    "@changesets/cli": "^2.25.2",
+    "@changesets/cli": "^2.26.2",
     "@open-wc/eslint-config": "^8.0.2",
-    "@tpluscode/eslint-config": "^0.3.0",
+    "@tpluscode/eslint-config": "^0.4.4",
     "@types/sparql-http-client": "^2.2.7",
-    "@typescript-eslint/eslint-plugin": "^5.35.1",
-    "@typescript-eslint/parser": "^5.35.1",
+    "@typescript-eslint/eslint-plugin": "^6.7.4",
+    "@typescript-eslint/parser": "^6.7.4",
     "c8": "^7.12.0",
     "chai-as-promised": "^7.1.1",
     "chai-snapshot-matcher": "^2.0.3",
@@ -58,6 +58,7 @@
     "eslint-plugin-n": "^15.2.5",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^6.0.0",
+    "eslint-plugin-require-extensions": "^0.1.3",
     "eslint-plugin-wc": "^1.3.2",
     "husky": "^8.0.1",
     "lint-staged": "^13.0.3",

diff --git a/packages/core/env.js b/packages/core/env.js
@@ -0,0 +1,6 @@
+import { createEnv } from '@rdfine/env'
+import { ShFactory } from '@rdfine/shacl/Factory'
+
+export default createEnv(
+  ShFactory,
+)
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -3,7 +3,9 @@
   "version": "0.0.0",
   "type": "module",
   "dependencies": {
+    "@rdfine/env": "^1.0.3",
+    "@rdfine/shacl": "^0.10.2",
     "@rdfjs/namespace": "^2.0.0",
-    "@zazuko/vocabulary-extras": "^1.2.3"
+    "@zazuko/vocabulary-extras": "^2.0.1"
   }
 }
diff --git a/packages/publish-views/lib/loadViews.js b/packages/publish-views/lib/loadViews.js
@@ -44,8 +44,10 @@ async function loadViewMeta(publishedView, metaObject, client) {
   const shape = await viewShape()
 
   const subjectVariable = 'view'
-  const query = shapeTo.construct(shape, { focusNode: metaObject, subjectVariable })
-  const dataset = await query.execute(client.query)
+  const query = shapeTo.constructQuery(shape, { focusNode: metaObject, subjectVariable })
+  const dataset = await query.execute(client.query, {
+    operation: 'postDirect',
+  })
 
   return toStream(dataset).pipe(through2.obj(viewIdTransform(metaObject, publishedView)))
 }

diff --git a/packages/publish-views/lib/shacl.js b/packages/publish-views/lib/shacl.js
@@ -5,11 +5,10 @@ import { ValidationError } from './ValidationError.js'
 const REPORTS_KEY = 'COMBINED_SHACL_REPORTS'
 
 export function collectShaclReports({ context, report }) {
-  const variables = context.variables.has(REPORTS_KEY)
-    ? context.variables
-    : context.variables.set(REPORTS_KEY, [])
+  context.variables.has(REPORTS_KEY) ||
+    context.variables.set(REPORTS_KEY, [])
 
-  variables.get(REPORTS_KEY).push(report)
+  context.variables.get(REPORTS_KEY).push(report)
 
   // continue stream
   return true
@@ -23,7 +22,7 @@ export function failOnAnyViolations() {
     this.push(chunk)
     next()
   }, function (done) {
-    const reports = variables.get(REPORTS_KEY)
+    const reports = variables.get(REPORTS_KEY, { allowMissing: true })
     if (reports?.some(hasViolations(ignoreWarnings))) {
       const error = new ValidationError(reports)
       logger.error(error.dataset.toString())

diff --git a/packages/publish-views/package.json b/packages/publish-views/package.json
@@ -14,18 +14,18 @@
     "test": "shacl-cli validate --shapes test/shape*.ttl --shapes ../../packages/core/shape/ViewValidationShape.ttl --data output/views.nt"
   },
   "dependencies": {
-    "@hydrofoil/shape-to-query": "^0.2.3",
-    "@tpluscode/rdf-ns-builders": "^2.0.1",
-    "@tpluscode/sparql-builder": "^0.3.24",
+    "@hydrofoil/shape-to-query": "^0.8.1",
+    "@tpluscode/rdf-ns-builders": "^4.1.0",
+    "@tpluscode/sparql-builder": "^1.1.0",
     "@view-builder/core": "^0.0.0",
     "@view-builder/view-util": "^0.0.1",
-    "barnard59": "^1.1.2",
-    "barnard59-base": "^1.1.0",
-    "barnard59-formats": "^1.2.0",
-    "barnard59-graph-store": "^1.0.1",
-    "barnard59-rdf": "^1.3.1",
-    "barnard59-validate-shacl": "^0.3.7",
-    "clownface": "^1.5.1",
+    "barnard59": "^3.0.2",
+    "barnard59-base": "^1.2.2",
+    "barnard59-formats": "^1.4.2",
+    "barnard59-graph-store": "^1.1.1",
+    "barnard59-rdf": "^1.4.5",
+    "barnard59-validate-shacl": "^0.3.8",
+    "clownface": "^2",
     "node-loader-raw": "^0.0.0",
     "onetime": "^6.0.0",
     "rdf-dataset-ext": "^1.0.1",

diff --git a/packages/publish-views/pipeline/to-file.ttl b/packages/publish-views/pipeline/to-file.ttl
@@ -7,7 +7,7 @@
     [
       :stepList
         (
-          <#loadViews> <#prepareView> <#pointerToDataset> <#validate> <#checkShaclReports> <#flatten> <#serialize> _:save
+          <Main> <#serialize> _:save
         )
     ] ;
   :variables

diff --git a/packages/view-util/package.json b/packages/view-util/package.json
@@ -5,17 +5,17 @@
   "main": "index.js",
   "dependencies": {
     "@rdfjs/term-map": "^2",
-    "@tpluscode/rdf-ns-builders": "^2.0.1",
-    "@tpluscode/sparql-builder": "^0.3.23",
+    "@tpluscode/rdf-ns-builders": "^4",
+    "@tpluscode/sparql-builder": "^1.1.0",
     "@view-builder/core": "0.0.0",
-    "clownface": "^1.5.1",
-    "rdf-cube-view-query": "^1.10.0",
+    "clownface": "^2",
+    "rdf-cube-view-query": "^1.12.0",
     "rdf-ext": "^2.0.0"
   },
   "devDependencies": {
     "@view-builder/testing": "0.0.0",
     "chai": "^4.3.6",
-    "is-graph-pointer": "^1.2.2",
+    "is-graph-pointer": "^2",
     "rdf-dataset-ext": "^1.0.1",
     "sinon": "^14.0.1",
     "sparql-http-client": "^2.4.1"