From 11cdb162bfa4f2c62bdae2e8b35e9eb5cb9ee904 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 13:51:21 +0200 Subject: [PATCH 01/21] `Development`: Upgrade markdown library to markdown-it --- package-lock.json | 83 ++++++++++++- package.json | 3 + ...rogramming-exercise-plant-uml.extension.ts | 115 ++++++++---------- .../programming-exercise-task.extension.ts | 28 +---- ...gramming-exercise-instruction.component.ts | 4 +- .../ArtemisTextReplacementExtension.ts | 19 +++ .../artemis-showdown-extension-wrapper.ts | 15 --- .../webapp/app/shared/markdown.service.ts | 6 +- .../shared/pipes/html-for-markdown.pipe.ts | 6 +- .../shared/util/markdown.conversion.util.ts | 55 ++++++--- 10 files changed, 200 insertions(+), 134 deletions(-) create mode 100644 src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension.ts delete mode 100644 src/main/webapp/app/shared/markdown-editor/extensions/artemis-showdown-extension-wrapper.ts diff --git a/package-lock.json b/package-lock.json index bb5825b0477e..b2a134fb5a23 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "@fortawesome/fontawesome-svg-core": "6.6.0", "@fortawesome/free-regular-svg-icons": "6.6.0", "@fortawesome/free-solid-svg-icons": "6.6.0", + "@iktakahiro/markdown-it-katex": "^4.0.1", "@ls1intum/apollon": "3.3.14", "@ng-bootstrap/ng-bootstrap": "17.0.1", "@ngx-translate/core": "15.0.0", @@ -37,6 +38,7 @@ "@sentry/angular": "8.30.0", "@swimlane/ngx-charts": "20.5.0", "@swimlane/ngx-graph": "8.4.0", + "@types/markdown-it": "^14.1.2", "@vscode/codicons": "0.0.36", "bootstrap": "5.3.3", "compare-versions": "6.1.1", @@ -54,6 +56,7 @@ "js-video-url-parser": "0.5.1", "jszip": "3.10.1", "lodash-es": "4.17.21", + "markdown-it": "^14.1.0", "mobile-drag-drop": "3.0.0-rc.0", "monaco-editor": "0.51.0", "ngx-infinite-scroll": "18.0.0", @@ -3573,6 +3576,15 @@ "react": "*" } }, + "node_modules/@iktakahiro/markdown-it-katex": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@iktakahiro/markdown-it-katex/-/markdown-it-katex-4.0.1.tgz", + "integrity": "sha512-kGFooO7fIOgY34PSG8ZNVsUlKhhNoqhzW2kq94TNGa8COzh73PO4KsEoPOsQVG1mEAe8tg7GqG0FoVao0aMHaw==", + "license": "MIT", + "dependencies": { + "katex": "^0.12.0" + } + }, "node_modules/@inquirer/checkbox": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-2.5.0.tgz", @@ -6561,6 +6573,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==", + "license": "MIT" + }, "node_modules/@types/lodash": { "version": "4.17.7", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.7.tgz", @@ -6578,6 +6596,22 @@ "@types/lodash": "*" } }, + "node_modules/@types/markdown-it": { + "version": "14.1.2", + "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-14.1.2.tgz", + "integrity": "sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==", + "license": "MIT", + "dependencies": { + "@types/linkify-it": "^5", + "@types/mdurl": "^2" + } + }, + "node_modules/@types/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==", + "license": "MIT" + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -7499,7 +7533,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, "license": "Python-2.0" }, "node_modules/aria-query": { @@ -10044,7 +10077,6 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "devOptional": true, "license": "BSD-2-Clause", "engines": { "node": ">=0.12" @@ -15257,6 +15289,15 @@ "dev": true, "license": "MIT" }, + "node_modules/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", + "license": "MIT", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, "node_modules/lint-staged": { "version": "15.2.10", "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-15.2.10.tgz", @@ -15919,12 +15960,35 @@ "tmpl": "1.0.5" } }, + "node_modules/markdown-it": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", + "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "bin": { + "markdown-it": "bin/markdown-it.mjs" + } + }, "node_modules/material-colors": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/material-colors/-/material-colors-1.2.6.tgz", "integrity": "sha512-6qE4B9deFBIa9YSpOc9O0Sgc43zTeVYbgDT5veRKSlB2+ZuHNoVVxA1L/ckMUayV9Ay9y7Z/SZCLcGteW9i7bg==", "license": "ISC" }, + "node_modules/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", + "license": "MIT" + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -18091,6 +18155,15 @@ "node": ">=6" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/pure-rand": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", @@ -20774,6 +20847,12 @@ "typescript-compare": "^0.0.2" } }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", + "license": "MIT" + }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", diff --git a/package.json b/package.json index 6bbcd55c3142..a90c669f44fb 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "@fortawesome/fontawesome-svg-core": "6.6.0", "@fortawesome/free-regular-svg-icons": "6.6.0", "@fortawesome/free-solid-svg-icons": "6.6.0", + "@iktakahiro/markdown-it-katex": "^4.0.1", "@ls1intum/apollon": "3.3.14", "@ng-bootstrap/ng-bootstrap": "17.0.1", "@ngx-translate/core": "15.0.0", @@ -40,6 +41,7 @@ "@sentry/angular": "8.30.0", "@swimlane/ngx-charts": "20.5.0", "@swimlane/ngx-graph": "8.4.0", + "@types/markdown-it": "^14.1.2", "@vscode/codicons": "0.0.36", "bootstrap": "5.3.3", "compare-versions": "6.1.1", @@ -57,6 +59,7 @@ "js-video-url-parser": "0.5.1", "jszip": "3.10.1", "lodash-es": "4.17.21", + "markdown-it": "^14.1.0", "mobile-drag-drop": "3.0.0-rc.0", "monaco-editor": "0.51.0", "ngx-infinite-scroll": "18.0.0", diff --git a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts index 2c0d4036369b..4a0f47c51210 100644 --- a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts +++ b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts @@ -1,20 +1,19 @@ import { Injectable } from '@angular/core'; import { ProgrammingExerciseTestCase } from 'app/entities/programming/programming-exercise-test-case.model'; +import { ArtemisTextReplacementExtension } from 'app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension'; +import { escapeStringForUseInRegex } from 'app/shared/util/global.utils'; import { Subject } from 'rxjs'; import { tap } from 'rxjs/operators'; -import { escapeStringForUseInRegex } from 'app/shared/util/global.utils'; import { ProgrammingExerciseInstructionService, TestCaseState } from 'app/exercises/programming/shared/instructions-render/service/programming-exercise-instruction.service'; import { ProgrammingExercisePlantUmlService } from 'app/exercises/programming/shared/instructions-render/service/programming-exercise-plant-uml.service'; -import { ArtemisShowdownExtensionWrapper } from 'app/shared/markdown-editor/extensions/artemis-showdown-extension-wrapper'; import { Result } from 'app/entities/result.model'; -import { ShowdownExtension } from 'showdown'; import DOMPurify from 'dompurify'; // This regex is the same as in the server: ProgrammingExerciseTaskService.java const testsColorRegex = /testsColor\((\s*[^()\s]+(\([^()]*\))?)\)/g; @Injectable({ providedIn: 'root' }) -export class ProgrammingExercisePlantUmlExtensionWrapper implements ArtemisShowdownExtensionWrapper { +export class ProgrammingExercisePlantUmlExtensionWrapper extends ArtemisTextReplacementExtension { private latestResult?: Result; private testCases?: ProgrammingExerciseTestCase[]; private injectableElementsFoundSubject = new Subject<() => void>(); @@ -25,7 +24,9 @@ export class ProgrammingExercisePlantUmlExtensionWrapper implements ArtemisShowd constructor( private programmingExerciseInstructionService: ProgrammingExerciseInstructionService, private plantUmlService: ProgrammingExercisePlantUmlService, - ) {} + ) { + super(); + } /** * Sets latest result according to parameter. @@ -66,65 +67,49 @@ export class ProgrammingExercisePlantUmlExtensionWrapper implements ArtemisShowd .subscribe(); } - /** - * Creates and returns an extension to current exercise. - * The extension provides a custom rendering mechanism for embedded plantUml diagrams. - * The mechanism works as follows: - * 1) Find (multiple) embedded plantUml diagrams based on a regex (startuml, enduml). - * 2) Replace the whole plantUml with a simple plantUml div container and a unique placeholder id - * 3) Add colors for test results in the plantUml (red, green, grey) - * 4) Send the plantUml content to the server for rendering a svg (the result will be cached for performance reasons) - * 5) Inject the computed svg for the plantUml (from the server) into the plantUml div container based on the unique placeholder id (see step 2) - */ - getExtension() { - const extension: ShowdownExtension = { - type: 'lang', - filter: (text: string) => { - const idPlaceholder = '%idPlaceholder%'; - // E.g. [task][Implement BubbleSort](testBubbleSort) - const plantUmlRegex = /@startuml([^@]*)@enduml/g; - // E.g. Implement BubbleSort, testBubbleSort - const plantUmlContainer = `
`; - // Replace test status markers. - const plantUmls = text.match(plantUmlRegex) ?? []; - // Assign unique ids to uml data structure at the beginning. - const plantUmlsIndexed = plantUmls.map((plantUml) => { - const nextIndex = this.plantUmlIndex; - // increase the global unique index so that the next plantUml gets a unique global id - this.plantUmlIndex++; - return { plantUmlId: nextIndex, plantUml }; - }); - // custom markdown to html rendering: replace the plantUml in the markdown with a simple
container with a unique id placeholder - // with the global unique id so that we can find the plantUml later on, when it was rendered, and then inject the 'actual' inner html (actually a svg image) - const replacedText = plantUmlsIndexed.reduce((acc: string, umlIndexed: { plantUmlId: number; plantUml: string }): string => { - return acc.replace(new RegExp(escapeStringForUseInRegex(umlIndexed.plantUml), 'g'), plantUmlContainer.replace(idPlaceholder, umlIndexed.plantUmlId.toString())); - }, text); - // before we send the plantUml to the server for rendering, we need to inject the current test status so that the colors can be adapted - // (green == implemented, red == not yet implemented, grey == unknown) - const plantUmlsValidated = plantUmlsIndexed.map((plantUmlIndexed: { plantUmlId: number; plantUml: string }) => { - plantUmlIndexed.plantUml = plantUmlIndexed.plantUml.replace(testsColorRegex, (match: any, capture: string) => { - const tests = this.programmingExerciseInstructionService.convertTestListToIds(capture, this.testCases); - const { testCaseState } = this.programmingExerciseInstructionService.testStatusForTask(tests, this.latestResult); - switch (testCaseState) { - case TestCaseState.SUCCESS: - return 'green'; - case TestCaseState.FAIL: - return 'red'; - default: - return 'grey'; - } - }); - return plantUmlIndexed; - }); - // send the adapted plantUml to the server for rendering and inject the result into the html DOM based on the unique plantUml id - this.injectableElementsFoundSubject.next(() => { - plantUmlsValidated.forEach((plantUmlIndexed: { plantUmlId: number; plantUml: string }) => { - this.loadAndInjectPlantUml(plantUmlIndexed.plantUml, plantUmlIndexed.plantUmlId); - }); - }); - return replacedText; - }, - }; - return extension; + replaceText(text: string): string { + const idPlaceholder = '%idPlaceholder%'; + // E.g. [task][Implement BubbleSort](testBubbleSort) + const plantUmlRegex = /@startuml([^@]*)@enduml/g; + // E.g. Implement BubbleSort, testBubbleSort + const plantUmlContainer = `
`; + // Replace test status markers. + const plantUmls = text.match(plantUmlRegex) ?? []; + // Assign unique ids to uml data structure at the beginning. + const plantUmlsIndexed = plantUmls.map((plantUml) => { + const nextIndex = this.plantUmlIndex; + // increase the global unique index so that the next plantUml gets a unique global id + this.plantUmlIndex++; + return { plantUmlId: nextIndex, plantUml }; + }); + // custom markdown to html rendering: replace the plantUml in the markdown with a simple
container with a unique id placeholder + // with the global unique id so that we can find the plantUml later on, when it was rendered, and then inject the 'actual' inner html (actually a svg image) + const replacedText = plantUmlsIndexed.reduce((acc: string, umlIndexed: { plantUmlId: number; plantUml: string }): string => { + return acc.replace(new RegExp(escapeStringForUseInRegex(umlIndexed.plantUml), 'g'), plantUmlContainer.replace(idPlaceholder, umlIndexed.plantUmlId.toString())); + }, text); + // before we send the plantUml to the server for rendering, we need to inject the current test status so that the colors can be adapted + // (green == implemented, red == not yet implemented, grey == unknown) + const plantUmlsValidated = plantUmlsIndexed.map((plantUmlIndexed: { plantUmlId: number; plantUml: string }) => { + plantUmlIndexed.plantUml = plantUmlIndexed.plantUml.replace(testsColorRegex, (match: any, capture: string) => { + const tests = this.programmingExerciseInstructionService.convertTestListToIds(capture, this.testCases); + const { testCaseState } = this.programmingExerciseInstructionService.testStatusForTask(tests, this.latestResult); + switch (testCaseState) { + case TestCaseState.SUCCESS: + return 'green'; + case TestCaseState.FAIL: + return 'red'; + default: + return 'grey'; + } + }); + return plantUmlIndexed; + }); + // send the adapted plantUml to the server for rendering and inject the result into the html DOM based on the unique plantUml id + this.injectableElementsFoundSubject.next(() => { + plantUmlsValidated.forEach((plantUmlIndexed: { plantUmlId: number; plantUml: string }) => { + this.loadAndInjectPlantUml(plantUmlIndexed.plantUml, plantUmlIndexed.plantUmlId); + }); + }); + return replacedText; } } diff --git a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts index 5e7b54444a14..c8c95eafb8b8 100644 --- a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts +++ b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts @@ -1,8 +1,7 @@ import { Injectable, ViewContainerRef } from '@angular/core'; import { TaskArrayWithExercise } from 'app/exercises/programming/shared/instructions-render/task/programming-exercise-task.model'; -import { ArtemisShowdownExtensionWrapper } from 'app/shared/markdown-editor/extensions/artemis-showdown-extension-wrapper'; +import { ArtemisTextReplacementExtension } from 'app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension'; import { Observable, Subject } from 'rxjs'; -import { ShowdownExtension } from 'showdown'; /** * Regular expression for finding tasks. @@ -18,15 +17,12 @@ import { ShowdownExtension } from 'showdown'; export const taskRegex = /\[task]\[([^[\]]+)]\(((?:[^(),]+(?:\([^()]*\)[^(),]*)?(?:,[^(),]+(?:\([^()]*\)[^(),]*)?)*)?)\)/g; @Injectable({ providedIn: 'root' }) -export class ProgrammingExerciseTaskExtensionWrapper implements ArtemisShowdownExtensionWrapper { +export class ProgrammingExerciseTaskExtensionWrapper extends ArtemisTextReplacementExtension { // We don't have a provider for ViewContainerRef, so we pass it from ProgrammingExerciseInstructionComponent viewContainerRef: ViewContainerRef; private testsForTaskSubject = new Subject(); private injectableElementsFoundSubject = new Subject<() => void>(); - - constructor() {} - /** * Subscribes to injectableElementsFoundSubject. */ @@ -34,24 +30,8 @@ export class ProgrammingExerciseTaskExtensionWrapper implements ArtemisShowdownE return this.injectableElementsFoundSubject.asObservable(); } - /** - * Creates and returns an extension to current exercise. - * The task regex is coupled to the value used in ProgrammingExerciseTaskService in the server and - * `TaskCommand` in the client - * If you change the regex, make sure to change it in all places! - */ - getExtension() { - const extension: ShowdownExtension = { - type: 'lang', - filter: (problemStatement: string) => { - return this.createTasks(problemStatement); - }, - }; - return extension; - } - - public createTasks(problemStatement: string): string { - return problemStatement.replace(taskRegex, (match) => { + replaceText(text: string): string { + return text.replace(taskRegex, (match) => { return this.escapeTaskSpecialCharactersForMarkdown(match); }); } diff --git a/src/main/webapp/app/exercises/programming/shared/instructions-render/programming-exercise-instruction.component.ts b/src/main/webapp/app/exercises/programming/shared/instructions-render/programming-exercise-instruction.component.ts index b0bd830496a9..9eac396fa2bc 100644 --- a/src/main/webapp/app/exercises/programming/shared/instructions-render/programming-exercise-instruction.component.ts +++ b/src/main/webapp/app/exercises/programming/shared/instructions-render/programming-exercise-instruction.component.ts @@ -16,7 +16,7 @@ import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; import { ThemeService } from 'app/core/theme/theme.service'; import { ProgrammingExerciseTestCase } from 'app/entities/programming/programming-exercise-test-case.model'; import { ProgrammingExerciseGradingService } from 'app/exercises/programming/manage/services/programming-exercise-grading.service'; -import { ShowdownExtension } from 'showdown'; +import type { PluginSimple } from 'markdown-it'; import { catchError, filter, map, mergeMap, switchMap, tap } from 'rxjs/operators'; import { Observable, Subscription, merge, of } from 'rxjs'; import { ProgrammingExercise } from 'app/entities/programming/programming-exercise.model'; @@ -80,7 +80,7 @@ export class ProgrammingExerciseInstructionComponent implements OnChanges, OnDes public renderedMarkdown: SafeHtml; private injectableContentForMarkdownCallbacks: Array<() => void> = []; - markdownExtensions: ShowdownExtension[]; + markdownExtensions: PluginSimple[]; private injectableContentFoundSubscription: Subscription; private tasksSubscription: Subscription; private generateHtmlSubscription: Subscription; diff --git a/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension.ts b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension.ts new file mode 100644 index 000000000000..d96d8c534bb0 --- /dev/null +++ b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension.ts @@ -0,0 +1,19 @@ +import type MarkdownIt from 'markdown-it'; +import type { PluginSimple } from 'markdown-it'; + +export abstract class ArtemisTextReplacementExtension { + getExtension(): PluginSimple { + return (md: MarkdownIt): void => { + // Override the `render` method to process the raw Markdown text before tokenizing + const originalRender = md.render.bind(md); + md.render = (markdownText: string, ...args) => { + // Perform the multi-line replacement on the raw markdown text + const modifiedText = this.replaceText(markdownText); + // Call the original render method with the modified text + return originalRender(modifiedText, ...args); + }; + }; + } + + abstract replaceText(text: string): string; +} diff --git a/src/main/webapp/app/shared/markdown-editor/extensions/artemis-showdown-extension-wrapper.ts b/src/main/webapp/app/shared/markdown-editor/extensions/artemis-showdown-extension-wrapper.ts deleted file mode 100644 index ce716b886c4e..000000000000 --- a/src/main/webapp/app/shared/markdown-editor/extensions/artemis-showdown-extension-wrapper.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { ShowdownExtension } from 'showdown'; -import { Observable } from 'rxjs'; - -/** - * The idea of this interface is to provide more information for an extension. - * By implementing the interface, the extension can use data that is in the closure of the class (e.g. this.latestResult). - * 1) The component that uses the extension can request it from the wrapper class by using getExtension. - * 2) In some cases it might also be necessary to inject content after the html is loaded, as async data fetching is necessary. - * Therefore, the component can subscribe for injectable elements. - * - */ -export interface ArtemisShowdownExtensionWrapper { - getExtension: () => ShowdownExtension; - subscribeForInjectableElementsFound: () => Observable<() => void>; -} diff --git a/src/main/webapp/app/shared/markdown.service.ts b/src/main/webapp/app/shared/markdown.service.ts index 646b03cb1b81..389fecba3eb9 100644 --- a/src/main/webapp/app/shared/markdown.service.ts +++ b/src/main/webapp/app/shared/markdown.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@angular/core'; import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; import { addCSSClass, htmlForMarkdown } from 'app/shared/util/markdown.conversion.util'; -import showdown from 'showdown'; +import type { PluginSimple } from 'markdown-it'; @Injectable({ providedIn: 'root' }) export class ArtemisMarkdownService { @@ -18,14 +18,14 @@ export class ArtemisMarkdownService { */ safeHtmlForMarkdown( markdownText?: string, - extensions: showdown.ShowdownExtension[] = [], + extensions: PluginSimple[] = [], allowedHtmlTags: string[] | undefined = undefined, allowedHtmlAttributes: string[] | undefined = undefined, ): SafeHtml { if (!markdownText || markdownText === '') { return ''; } - const convertedString = htmlForMarkdown(markdownText, [...extensions, ...addCSSClass], allowedHtmlTags, allowedHtmlAttributes); + const convertedString = htmlForMarkdown(markdownText, [...extensions, addCSSClass], allowedHtmlTags, allowedHtmlAttributes); return this.sanitizer.bypassSecurityTrustHtml(convertedString); } diff --git a/src/main/webapp/app/shared/pipes/html-for-markdown.pipe.ts b/src/main/webapp/app/shared/pipes/html-for-markdown.pipe.ts index 610faf20c0ca..660a5c0971dc 100644 --- a/src/main/webapp/app/shared/pipes/html-for-markdown.pipe.ts +++ b/src/main/webapp/app/shared/pipes/html-for-markdown.pipe.ts @@ -1,7 +1,7 @@ import { Pipe, PipeTransform } from '@angular/core'; -import { ShowdownExtension } from 'showdown'; import { SafeHtml } from '@angular/platform-browser'; import { ArtemisMarkdownService } from 'app/shared/markdown.service'; +import type { PluginSimple } from 'markdown-it'; @Pipe({ name: 'htmlForMarkdown', @@ -12,14 +12,14 @@ export class HtmlForMarkdownPipe implements PipeTransform { /** * Converts markdown into html, sanitizes it and then declares it as safe to bypass further security. * @param {string} markdown the original markdown text - * @param {ShowdownExtension[]} extensions to use for markdown parsing + * @param {PluginSimple[]} extensions to use for markdown parsing * @param {string[]} allowedHtmlTags to allow during sanitization * @param {string[]} allowedHtmlAttributes to allow during sanitization * @returns {string} the resulting html as a SafeHtml object that can be inserted into the angular template */ transform( markdown?: string, - extensions: ShowdownExtension[] = [], + extensions: PluginSimple[] = [], allowedHtmlTags: string[] | undefined = undefined, allowedHtmlAttributes: string[] | undefined = undefined, ): SafeHtml { diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index 658574db088b..c300e9b22b31 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -1,7 +1,7 @@ import showdown from 'showdown'; -import showdownKatex from 'showdown-katex'; -import showdownHighlight from 'showdown-highlight'; import DOMPurify, { Config } from 'dompurify'; +import type { PluginSimple } from 'markdown-it'; +import markdownit from 'markdown-it'; /** * showdown will add the classes to the converted html @@ -12,13 +12,16 @@ const classMap: { [key: string]: string } = { }; /** * extension to add css classes to html tags - * see: https://github.com/showdownjs/showdown/wiki/Add-default-classes-for-each-HTML-element */ -export const addCSSClass = Object.keys(classMap).map((key) => ({ - type: 'output', - regex: new RegExp(`<${key}(.*)>`, 'g'), - replace: `<${key} class="${classMap[key]}" $1>`, -})); +export const addCSSClass: PluginSimple = (md) => { + for (const key in classMap) { + const originalRender = md.renderer.rules[key] || md.renderer.rules.defaultRender; + md.renderer.rules[key] = (tokens, idx, options, env, self) => { + tokens[idx].attrPush(['class', classMap[key]]); + return originalRender ? originalRender(tokens, idx, options, env, self) : self.renderToken(tokens, idx, options); + }; + } +}; /** * Converts markdown into html (string) and sanitizes it. Does NOT declare it as safe to bypass further security @@ -32,13 +35,35 @@ export const addCSSClass = Object.keys(classMap).map((key) => ({ */ export function htmlForMarkdown( markdownText?: string, - extensions: showdown.ShowdownExtension[] = [], + extensions: PluginSimple[] = [], allowedHtmlTags: string[] | undefined = undefined, allowedHtmlAttributes: string[] | undefined = undefined, ): string { if (!markdownText || markdownText === '') { return ''; } + + const purifyParameters = {} as Config; + // Prevents sanitizer from deleting id or PlantUML color tags + purifyParameters['ADD_TAGS'] = ['testid', 'color:grey', 'color:green', 'color:red']; + if (allowedHtmlTags) { + purifyParameters['ALLOWED_TAGS'] = allowedHtmlTags; + } + if (allowedHtmlAttributes) { + purifyParameters['ALLOWED_ATTR'] = allowedHtmlAttributes; + } + const html = DOMPurify.sanitize(markdownText, purifyParameters) as string; + + let md = markdownit({ + html: true, + linkify: true, + // TODO code highlight, katex, etc + }); + for (const extension of extensions) { + md = md.use(extension); + } + return md.render(html); + /* const converter = new showdown.Converter({ parseImgDimensions: true, headerLevelStart: 3, @@ -49,17 +74,7 @@ export function htmlForMarkdown( backslashEscapesHTMLTags: true, extensions: [...extensions, showdownKatex(), showdownHighlight({ pre: true }), ...addCSSClass], }); - const html = converter.makeHtml(markdownText); - const purifyParameters = {} as Config; - // Prevents sanitizer from deleting id - purifyParameters['ADD_TAGS'] = ['testid']; - if (allowedHtmlTags) { - purifyParameters['ALLOWED_TAGS'] = allowedHtmlTags; - } - if (allowedHtmlAttributes) { - purifyParameters['ALLOWED_ATTR'] = allowedHtmlAttributes; - } - return DOMPurify.sanitize(html, purifyParameters) as string; + */ } export function markdownForHtml(htmlText: string): string { From fbe63f7ca3c32033d7f0459cc6c85c218b882fc0 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 14:02:45 +0200 Subject: [PATCH 02/21] fix plantUML rendering --- .../shared/util/markdown.conversion.util.ts | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index c300e9b22b31..55ed0d4803c9 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -1,7 +1,7 @@ -import showdown from 'showdown'; import DOMPurify, { Config } from 'dompurify'; import type { PluginSimple } from 'markdown-it'; import markdownit from 'markdown-it'; +import showdown from 'showdown'; /** * showdown will add the classes to the converted html @@ -43,17 +43,6 @@ export function htmlForMarkdown( return ''; } - const purifyParameters = {} as Config; - // Prevents sanitizer from deleting id or PlantUML color tags - purifyParameters['ADD_TAGS'] = ['testid', 'color:grey', 'color:green', 'color:red']; - if (allowedHtmlTags) { - purifyParameters['ALLOWED_TAGS'] = allowedHtmlTags; - } - if (allowedHtmlAttributes) { - purifyParameters['ALLOWED_ATTR'] = allowedHtmlAttributes; - } - const html = DOMPurify.sanitize(markdownText, purifyParameters) as string; - let md = markdownit({ html: true, linkify: true, @@ -62,7 +51,22 @@ export function htmlForMarkdown( for (const extension of extensions) { md = md.use(extension); } - return md.render(html); + + const mdtext = md.render(markdownText); + + const purifyParameters = {} as Config; + // Prevents sanitizer from deleting id + purifyParameters['ADD_TAGS'] = ['testid']; + if (allowedHtmlTags) { + purifyParameters['ALLOWED_TAGS'] = allowedHtmlTags; + } + if (allowedHtmlAttributes) { + purifyParameters['ALLOWED_ATTR'] = allowedHtmlAttributes; + } + const html = DOMPurify.sanitize(mdtext, purifyParameters) as string; + + //return md.render(html); + return html; /* const converter = new showdown.Converter({ parseImgDimensions: true, From 4be890e6b5f37a54b24fc64e22a927e629f4b242 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 15:10:18 +0200 Subject: [PATCH 03/21] fix tables --- package-lock.json | 21 +++++++++++-- package.json | 6 ++-- .../webapp/app/shared/markdown.service.ts | 4 +-- .../shared/util/markdown.conversion.util.ts | 31 ++++++++----------- 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/package-lock.json b/package-lock.json index b2a134fb5a23..a77a7bd7f086 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,7 +30,7 @@ "@fortawesome/fontawesome-svg-core": "6.6.0", "@fortawesome/free-regular-svg-icons": "6.6.0", "@fortawesome/free-solid-svg-icons": "6.6.0", - "@iktakahiro/markdown-it-katex": "^4.0.1", + "@iktakahiro/markdown-it-katex": "4.0.1", "@ls1intum/apollon": "3.3.14", "@ng-bootstrap/ng-bootstrap": "17.0.1", "@ngx-translate/core": "15.0.0", @@ -56,7 +56,9 @@ "js-video-url-parser": "0.5.1", "jszip": "3.10.1", "lodash-es": "4.17.21", - "markdown-it": "^14.1.0", + "markdown-it": "14.1.0", + "markdown-it-class": "1.0.0", + "markdown-it-highlightjs": "4.2.0", "mobile-drag-drop": "3.0.0-rc.0", "monaco-editor": "0.51.0", "ngx-infinite-scroll": "18.0.0", @@ -15977,6 +15979,21 @@ "markdown-it": "bin/markdown-it.mjs" } }, + "node_modules/markdown-it-class": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/markdown-it-class/-/markdown-it-class-1.0.0.tgz", + "integrity": "sha512-CVDYqSgmErLAqInwWu8WmAR2nX6MMIBIt8LB6qg8DNldca9+aoC6ZyuY0lvBMsaTSHNFJRkcHVR1XjLw9nr9qQ==", + "license": "MIT" + }, + "node_modules/markdown-it-highlightjs": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/markdown-it-highlightjs/-/markdown-it-highlightjs-4.2.0.tgz", + "integrity": "sha512-NC7pXE8KkOl6xWJVRNt8p6wgJVznXKsE0HgYGdk6DD2tn1l4L9f0ALf3VIoGVkotNU1uGQatSxfBF1zZPUMmuQ==", + "license": "Unlicense", + "dependencies": { + "highlight.js": "^11.9.0" + } + }, "node_modules/material-colors": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/material-colors/-/material-colors-1.2.6.tgz", diff --git a/package.json b/package.json index a90c669f44fb..dee57aeba93f 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "@fortawesome/fontawesome-svg-core": "6.6.0", "@fortawesome/free-regular-svg-icons": "6.6.0", "@fortawesome/free-solid-svg-icons": "6.6.0", - "@iktakahiro/markdown-it-katex": "^4.0.1", + "@iktakahiro/markdown-it-katex": "4.0.1", "@ls1intum/apollon": "3.3.14", "@ng-bootstrap/ng-bootstrap": "17.0.1", "@ngx-translate/core": "15.0.0", @@ -59,7 +59,9 @@ "js-video-url-parser": "0.5.1", "jszip": "3.10.1", "lodash-es": "4.17.21", - "markdown-it": "^14.1.0", + "markdown-it": "14.1.0", + "markdown-it-class": "1.0.0", + "markdown-it-highlightjs": "4.2.0", "mobile-drag-drop": "3.0.0-rc.0", "monaco-editor": "0.51.0", "ngx-infinite-scroll": "18.0.0", diff --git a/src/main/webapp/app/shared/markdown.service.ts b/src/main/webapp/app/shared/markdown.service.ts index 389fecba3eb9..c7feb49eb4a0 100644 --- a/src/main/webapp/app/shared/markdown.service.ts +++ b/src/main/webapp/app/shared/markdown.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@angular/core'; import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; -import { addCSSClass, htmlForMarkdown } from 'app/shared/util/markdown.conversion.util'; +import { htmlForMarkdown } from 'app/shared/util/markdown.conversion.util'; import type { PluginSimple } from 'markdown-it'; @Injectable({ providedIn: 'root' }) @@ -25,7 +25,7 @@ export class ArtemisMarkdownService { if (!markdownText || markdownText === '') { return ''; } - const convertedString = htmlForMarkdown(markdownText, [...extensions, addCSSClass], allowedHtmlTags, allowedHtmlAttributes); + const convertedString = htmlForMarkdown(markdownText, extensions, allowedHtmlTags, allowedHtmlAttributes); return this.sanitizer.bypassSecurityTrustHtml(convertedString); } diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index 55ed0d4803c9..4b90ff1ac7e1 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -1,28 +1,20 @@ import DOMPurify, { Config } from 'dompurify'; import type { PluginSimple } from 'markdown-it'; -import markdownit from 'markdown-it'; +import markdownIt from 'markdown-it'; import showdown from 'showdown'; +import markdown_it_highlightjs from 'markdown-it-highlightjs'; +// These libraries are not typed +// @ts-expect-error +import markdownItClass from 'markdown-it-class'; +// @ts-expect-error +import markdownItKatex from '@iktakahiro/markdown-it-katex'; /** - * showdown will add the classes to the converted html - * see: https://github.com/showdownjs/showdown/wiki/Add-default-classes-for-each-HTML-element + * Add these classes to the converted html. */ const classMap: { [key: string]: string } = { table: 'table', }; -/** - * extension to add css classes to html tags - */ -export const addCSSClass: PluginSimple = (md) => { - for (const key in classMap) { - const originalRender = md.renderer.rules[key] || md.renderer.rules.defaultRender; - md.renderer.rules[key] = (tokens, idx, options, env, self) => { - tokens[idx].attrPush(['class', classMap[key]]); - return originalRender ? originalRender(tokens, idx, options, env, self) : self.renderToken(tokens, idx, options); - }; - } -}; - /** * Converts markdown into html (string) and sanitizes it. Does NOT declare it as safe to bypass further security * Note: If possible, please use safeHtmlForMarkdown @@ -43,15 +35,18 @@ export function htmlForMarkdown( return ''; } - let md = markdownit({ + let md = markdownIt({ html: true, linkify: true, - // TODO code highlight, katex, etc + breaks: true, }); for (const extension of extensions) { md = md.use(extension); } + // Add default extensions (Code Highlight, Latex) + md = md.use(markdown_it_highlightjs).use(markdownItKatex).use(markdownItClass, classMap); + const mdtext = md.render(markdownText); const purifyParameters = {} as Config; From f791e11ae8a40ecb6d9adfec07d16614aa953c92 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 15:27:23 +0200 Subject: [PATCH 04/21] remove showdown --- angular.json | 5 +- package-lock.json | 115 +++++------------- package.json | 7 +- .../shared/util/markdown.conversion.util.ts | 33 +---- 4 files changed, 38 insertions(+), 122 deletions(-) diff --git a/angular.json b/angular.json index e5543ff2ce60..b77e49684482 100644 --- a/angular.json +++ b/angular.json @@ -44,9 +44,8 @@ "react-is", "rfdc", "shallowequal", - "showdown-highlight", - "showdown-katex", - "showdown", + "markdown-it-class", + "@iktakahiro/markdown-it-katex", "smoothscroll-polyfill", "sockjs-client", "use-sync-external-store/shim", diff --git a/package-lock.json b/package-lock.json index a77a7bd7f086..800519b1354b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,7 +38,6 @@ "@sentry/angular": "8.30.0", "@swimlane/ngx-charts": "20.5.0", "@swimlane/ngx-graph": "8.4.0", - "@types/markdown-it": "^14.1.2", "@vscode/codicons": "0.0.36", "bootstrap": "5.3.3", "compare-versions": "6.1.1", @@ -67,15 +66,13 @@ "pdfjs-dist": "4.6.82", "posthog-js": "1.161.6", "rxjs": "7.8.1", - "showdown": "2.1.0", - "showdown-highlight": "3.1.0", - "showdown-katex": "0.6.0", "simple-statistics": "7.8.5", "smoothscroll-polyfill": "0.4.4", "sockjs-client": "1.6.1", "split.js": "1.6.5", "ts-cacheable": "1.0.10", "tslib": "2.7.0", + "turndown": "7.2.0", "uuid": "10.0.0", "webstomp-client": "1.2.6", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", @@ -98,11 +95,13 @@ "@types/dompurify": "3.0.5", "@types/jest": "29.5.13", "@types/lodash-es": "4.17.12", + "@types/markdown-it": "14.1.2", "@types/node": "22.5.5", "@types/papaparse": "5.3.14", "@types/showdown": "2.0.6", "@types/smoothscroll-polyfill": "0.3.4", "@types/sockjs-client": "1.5.4", + "@types/turndown": "5.0.5", "@types/uuid": "10.0.0", "@typescript-eslint/eslint-plugin": "8.6.0", "@typescript-eslint/parser": "8.6.0", @@ -5138,6 +5137,12 @@ "node": ">=6" } }, + "node_modules/@mixmark-io/domino": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", + "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", + "license": "BSD-2-Clause" + }, "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", @@ -6579,6 +6584,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-5.0.0.tgz", "integrity": "sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==", + "dev": true, "license": "MIT" }, "node_modules/@types/lodash": { @@ -6602,6 +6608,7 @@ "version": "14.1.2", "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-14.1.2.tgz", "integrity": "sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==", + "dev": true, "license": "MIT", "dependencies": { "@types/linkify-it": "^5", @@ -6612,6 +6619,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-2.0.0.tgz", "integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==", + "dev": true, "license": "MIT" }, "node_modules/@types/mime": { @@ -6790,6 +6798,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/turndown": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.5.tgz", + "integrity": "sha512-TL2IgGgc7B5j78rIccBtlYAnkuv8nUQqhQc+DSYV5j9Be9XOcm/SKOVRuA47xAVI3680Tk9B1d8flK2GWT2+4w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/use-sync-external-store": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz", @@ -11934,15 +11949,6 @@ "node": ">= 0.4" } }, - "node_modules/he": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "license": "MIT", - "bin": { - "he": "bin/he" - } - }, "node_modules/highlight.js": { "version": "11.10.0", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.10.0.tgz", @@ -12006,17 +12012,6 @@ "integrity": "sha512-9SQg9oLQSAOZb8rO17mRNPkVB95QRh6iLY5J0Dbc/cgeoBT+XJBK/6XrQqfd+vxUVRjdctW+sfgYqgYzi0vg9g==", "license": "ISC" }, - "node_modules/html-encoder-decoder": { - "version": "1.3.10", - "resolved": "https://registry.npmjs.org/html-encoder-decoder/-/html-encoder-decoder-1.3.10.tgz", - "integrity": "sha512-18SjgzQZ9U1mxb96rjcWgWMnTlEzNj2lU2wAU7OeUobdIWXTS6lOGc6419eLhMlX24sNQYDyQfgkSXWjyq/Ilg==", - "license": "MIT", - "dependencies": { - "he": "^1.1.0", - "iterate-object": "^1.3.2", - "regex-escape": "^3.4.2" - } - }, "node_modules/html-encoding-sniffer": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz", @@ -12885,12 +12880,6 @@ "node": ">=8" } }, - "node_modules/iterate-object": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/iterate-object/-/iterate-object-1.3.4.tgz", - "integrity": "sha512-4dG1D1x/7g8PwHS9aK6QV5V94+ZvyP4+d19qDv43EzImmrndysIl4prmJ1hWWIGCqrZHyaHBm6BSEWHOLnpoNw==", - "license": "MIT" - }, "node_modules/jackspeak": { "version": "3.4.3", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", @@ -18517,12 +18506,6 @@ "@babel/runtime": "^7.8.4" } }, - "node_modules/regex-escape": { - "version": "3.4.10", - "resolved": "https://registry.npmjs.org/regex-escape/-/regex-escape-3.4.10.tgz", - "integrity": "sha512-qEqf7uzW+iYcKNLMDFnMkghhQBnGdivT6KqVQyKsyjSWnoFyooXVnxrw9dtv3AFLnD6VBGXxtZGAQNFGFTnCqA==", - "license": "MIT" - }, "node_modules/regex-parser": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.3.0.tgz", @@ -19385,57 +19368,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/showdown": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/showdown/-/showdown-2.1.0.tgz", - "integrity": "sha512-/6NVYu4U819R2pUIk79n67SYgJHWCce0a5xTP979WbNp0FL9MN1I1QK662IDU1b6JzKTvmhgI7T7JYIxBi3kMQ==", - "license": "MIT", - "dependencies": { - "commander": "^9.0.0" - }, - "bin": { - "showdown": "bin/showdown.js" - }, - "funding": { - "type": "individual", - "url": "https://www.paypal.me/tiviesantos" - } - }, - "node_modules/showdown-highlight": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/showdown-highlight/-/showdown-highlight-3.1.0.tgz", - "integrity": "sha512-wrTxtE63L/bpW5A2Uy/AO1gblXnNHK/cDL6LszECOoCdMJKWTj0/4n4I/pmqub+3H3KCPVDDvtXpCArnT/heFA==", - "license": "MIT", - "dependencies": { - "highlight.js": "^11.5.0", - "html-encoder-decoder": "^1.3.9", - "showdown": "^2.0.3" - } - }, - "node_modules/showdown-katex": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/showdown-katex/-/showdown-katex-0.6.0.tgz", - "integrity": "sha512-eEOipJjqMxRJ+e69WlA7XENhFZzKhNl12csey0iLd4QbLzGF61+FBxNPhEZFz9wICYTJNfyqNgLSqmm8Uj0fGA==", - "license": "MIT", - "dependencies": { - "katex": "^0.10.0" - }, - "engines": { - "node": "*" - }, - "peerDependencies": { - "showdown": "^1.4.3" - } - }, - "node_modules/showdown/node_modules/commander": { - "version": "9.5.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", - "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", - "license": "MIT", - "engines": { - "node": "^12.20.0 || >=14" - } - }, "node_modules/side-channel": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", @@ -20770,6 +20702,15 @@ "node": "^16.14.0 || >=18.0.0" } }, + "node_modules/turndown": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.0.tgz", + "integrity": "sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A==", + "license": "MIT", + "dependencies": { + "@mixmark-io/domino": "^2.2.0" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index dee57aeba93f..1d90e03d8a5b 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,6 @@ "@sentry/angular": "8.30.0", "@swimlane/ngx-charts": "20.5.0", "@swimlane/ngx-graph": "8.4.0", - "@types/markdown-it": "^14.1.2", "@vscode/codicons": "0.0.36", "bootstrap": "5.3.3", "compare-versions": "6.1.1", @@ -70,15 +69,13 @@ "pdfjs-dist": "4.6.82", "posthog-js": "1.161.6", "rxjs": "7.8.1", - "showdown": "2.1.0", - "showdown-highlight": "3.1.0", - "showdown-katex": "0.6.0", "simple-statistics": "7.8.5", "smoothscroll-polyfill": "0.4.4", "sockjs-client": "1.6.1", "split.js": "1.6.5", "ts-cacheable": "1.0.10", "tslib": "2.7.0", + "turndown": "7.2.0", "uuid": "10.0.0", "webstomp-client": "1.2.6", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", @@ -136,11 +133,13 @@ "@types/dompurify": "3.0.5", "@types/jest": "29.5.13", "@types/lodash-es": "4.17.12", + "@types/markdown-it": "14.1.2", "@types/node": "22.5.5", "@types/papaparse": "5.3.14", "@types/showdown": "2.0.6", "@types/smoothscroll-polyfill": "0.3.4", "@types/sockjs-client": "1.5.4", + "@types/turndown": "5.0.5", "@types/uuid": "10.0.0", "@typescript-eslint/eslint-plugin": "8.6.0", "@typescript-eslint/parser": "8.6.0", diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index 4b90ff1ac7e1..78935c4f5d30 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -1,13 +1,13 @@ import DOMPurify, { Config } from 'dompurify'; import type { PluginSimple } from 'markdown-it'; import markdownIt from 'markdown-it'; -import showdown from 'showdown'; -import markdown_it_highlightjs from 'markdown-it-highlightjs'; // These libraries are not typed // @ts-expect-error import markdownItClass from 'markdown-it-class'; // @ts-expect-error import markdownItKatex from '@iktakahiro/markdown-it-katex'; +import markdown_it_highlightjs from 'markdown-it-highlightjs'; +import TurndownService from 'turndown'; /** * Add these classes to the converted html. @@ -58,33 +58,10 @@ export function htmlForMarkdown( if (allowedHtmlAttributes) { purifyParameters['ALLOWED_ATTR'] = allowedHtmlAttributes; } - const html = DOMPurify.sanitize(mdtext, purifyParameters) as string; - - //return md.render(html); - return html; - /* - const converter = new showdown.Converter({ - parseImgDimensions: true, - headerLevelStart: 3, - simplifiedAutoLink: true, - strikethrough: true, - tables: true, - openLinksInNewWindow: true, - backslashEscapesHTMLTags: true, - extensions: [...extensions, showdownKatex(), showdownHighlight({ pre: true }), ...addCSSClass], - }); - */ + return DOMPurify.sanitize(mdtext, purifyParameters) as string; } export function markdownForHtml(htmlText: string): string { - const converter = new showdown.Converter({ - parseImgDimensions: true, - headerLevelStart: 3, - simplifiedAutoLink: true, - strikethrough: true, - tables: true, - openLinksInNewWindow: true, - backslashEscapesHTMLTags: true, - }); - return converter.makeMarkdown(htmlText); + const turndownService = new TurndownService(); + return turndownService.turndown(htmlText); } From 70c6e0181310b8cf1fd7ef7a3c52909919c77a9b Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 15:37:14 +0200 Subject: [PATCH 05/21] Improve comments --- .../programming-exercise-plant-uml.extension.ts | 9 +++++++++ .../extensions/programming-exercise-task.extension.ts | 5 +++++ .../extensions/ArtemisTextReplacementExtension.ts | 5 ++++- .../webapp/app/shared/util/markdown.conversion.util.ts | 5 ++--- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts index 4a0f47c51210..9aab41e422cc 100644 --- a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts +++ b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts @@ -67,6 +67,15 @@ export class ProgrammingExercisePlantUmlExtensionWrapper extends ArtemisTextRepl .subscribe(); } + /** + * The extension provides a custom rendering mechanism for embedded plantUml diagrams. + * The mechanism works as follows: + * 1) Find (multiple) embedded plantUml diagrams based on a regex (startuml, enduml). + * 2) Replace the whole plantUml with a simple plantUml div container and a unique placeholder id + * 3) Add colors for test results in the plantUml (red, green, grey) + * 4) Send the plantUml content to the server for rendering a svg (the result will be cached for performance reasons) + * 5) Inject the computed svg for the plantUml (from the server) into the plantUml div container based on the unique placeholder id (see step 2) + */ replaceText(text: string): string { const idPlaceholder = '%idPlaceholder%'; // E.g. [task][Implement BubbleSort](testBubbleSort) diff --git a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts index c8c95eafb8b8..d3157bf30d50 100644 --- a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts +++ b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts @@ -30,6 +30,11 @@ export class ProgrammingExerciseTaskExtensionWrapper extends ArtemisTextReplacem return this.injectableElementsFoundSubject.asObservable(); } + /** + * The task regex is coupled to the value used in ProgrammingExerciseTaskService in the server + * and `TaskCommand` in the client + * If you change the regex, make sure to change it in all places! + */ replaceText(text: string): string { return text.replace(taskRegex, (match) => { return this.escapeTaskSpecialCharactersForMarkdown(match); diff --git a/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension.ts b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension.ts index d96d8c534bb0..51b8a273d2e7 100644 --- a/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension.ts +++ b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension.ts @@ -1,13 +1,16 @@ import type MarkdownIt from 'markdown-it'; import type { PluginSimple } from 'markdown-it'; +/** + * Markdown-It extension that allows replacing text in the raw markdown before tokenizing. + */ export abstract class ArtemisTextReplacementExtension { getExtension(): PluginSimple { return (md: MarkdownIt): void => { // Override the `render` method to process the raw Markdown text before tokenizing const originalRender = md.render.bind(md); md.render = (markdownText: string, ...args) => { - // Perform the multi-line replacement on the raw markdown text + // Perform the replacement on the raw markdown text const modifiedText = this.replaceText(markdownText); // Call the original render method with the modified text return originalRender(modifiedText, ...args); diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index 78935c4f5d30..0fb1b4b21453 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -1,10 +1,9 @@ import DOMPurify, { Config } from 'dompurify'; import type { PluginSimple } from 'markdown-it'; import markdownIt from 'markdown-it'; -// These libraries are not typed -// @ts-expect-error +// @ts-expect-error library is not typed import markdownItClass from 'markdown-it-class'; -// @ts-expect-error +// @ts-expect-error library is not typed import markdownItKatex from '@iktakahiro/markdown-it-katex'; import markdown_it_highlightjs from 'markdown-it-highlightjs'; import TurndownService from 'turndown'; From 279b1e641553f02ea5f3a1ece68f2a162a839e68 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 15:39:08 +0200 Subject: [PATCH 06/21] update typing --- src/main/webapp/app/index.d.ts | 8 ++++---- .../webapp/app/shared/util/markdown.conversion.util.ts | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/webapp/app/index.d.ts b/src/main/webapp/app/index.d.ts index 44ce332f5e6b..ce7cfe74d6b5 100644 --- a/src/main/webapp/app/index.d.ts +++ b/src/main/webapp/app/index.d.ts @@ -1,9 +1,9 @@ -declare module 'showdown-katex' { - const main: () => ShowDownExtension; +declare module 'markdown-it-class' { + const main: (md: MarkdownIt) => void; export = main; } -declare module 'showdown-highlight' { - const main: ({ pre: boolean }) => ShowDownExtension; +declare module '@iktakahiro/markdown-it-katex' { + const main: (md: MarkdownIt) => void; export = main; } diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index 0fb1b4b21453..ec9f461460a4 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -1,9 +1,7 @@ import DOMPurify, { Config } from 'dompurify'; import type { PluginSimple } from 'markdown-it'; import markdownIt from 'markdown-it'; -// @ts-expect-error library is not typed import markdownItClass from 'markdown-it-class'; -// @ts-expect-error library is not typed import markdownItKatex from '@iktakahiro/markdown-it-katex'; import markdown_it_highlightjs from 'markdown-it-highlightjs'; import TurndownService from 'turndown'; From 185cbb4c2f4f34144e21c09248c380b0ef5104cc Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 17:24:06 +0200 Subject: [PATCH 07/21] fix client test --- .../quiz/shared/short-answer-question-util.service.ts | 8 ++++++++ .../short-answer-question-util.service.spec.ts | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/app/exercises/quiz/shared/short-answer-question-util.service.ts b/src/main/webapp/app/exercises/quiz/shared/short-answer-question-util.service.ts index c9cf7bc98076..155feb09a5f5 100644 --- a/src/main/webapp/app/exercises/quiz/shared/short-answer-question-util.service.ts +++ b/src/main/webapp/app/exercises/quiz/shared/short-answer-question-util.service.ts @@ -373,6 +373,14 @@ export class ShortAnswerQuestionUtil { if (firstWord === '') { continue; } + // Remove ending newline character + for (let j = 0; j < formattedTextParts[i].length; j++) { + const spot = formattedTextParts[i][j]; + if (spot.endsWith('\n')) { + formattedTextParts[i][j] = spot.substring(0, spot.length - 1); + } + } + const firstWordIndex = element.indexOf(firstWord); const whitespace = ' '.repeat(this.getIndentation(originalTextParts[i][0]).length); formattedTextParts[i][0] = [element.substring(0, firstWordIndex), whitespace, element.substring(firstWordIndex).trim()].join(''); diff --git a/src/test/javascript/spec/component/short-answer-quiz/short-answer-question-util.service.spec.ts b/src/test/javascript/spec/component/short-answer-quiz/short-answer-question-util.service.spec.ts index 33fdbbc3ba71..275095227749 100644 --- a/src/test/javascript/spec/component/short-answer-quiz/short-answer-question-util.service.spec.ts +++ b/src/test/javascript/spec/component/short-answer-quiz/short-answer-question-util.service.spec.ts @@ -17,7 +17,7 @@ describe('ShortAnswerQuestionUtil', () => { const originalTextParts2 = [['`random code`'], ['` some more code`', '[-spot 1]'], ['`last code paragraph`']]; const formattedTextParts2 = [ ['

random code

'], - ['

    some more code

', '

[-spot 1]

'], + ['

    some more code

', '

[-spot 1]

'], ['

last code paragraph

'], ]; expect(shortAnswerQuestionUtil.transformTextPartsIntoHTML(originalTextParts2)).toEqual(formattedTextParts2); From 5527c89a3d4b0abc6de63523f2d3da90944875a5 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 17:52:08 +0200 Subject: [PATCH 08/21] remove newline --- .../quiz/shared/short-answer-question-util.service.ts | 7 ------- .../webapp/app/shared/util/markdown.conversion.util.ts | 9 +++++++-- .../posting-content-part.component.spec.ts | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/main/webapp/app/exercises/quiz/shared/short-answer-question-util.service.ts b/src/main/webapp/app/exercises/quiz/shared/short-answer-question-util.service.ts index 155feb09a5f5..6e6aac7e24eb 100644 --- a/src/main/webapp/app/exercises/quiz/shared/short-answer-question-util.service.ts +++ b/src/main/webapp/app/exercises/quiz/shared/short-answer-question-util.service.ts @@ -373,13 +373,6 @@ export class ShortAnswerQuestionUtil { if (firstWord === '') { continue; } - // Remove ending newline character - for (let j = 0; j < formattedTextParts[i].length; j++) { - const spot = formattedTextParts[i][j]; - if (spot.endsWith('\n')) { - formattedTextParts[i][j] = spot.substring(0, spot.length - 1); - } - } const firstWordIndex = element.indexOf(firstWord); const whitespace = ' '.repeat(this.getIndentation(originalTextParts[i][0]).length); diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index ec9f461460a4..998abb524607 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -44,7 +44,12 @@ export function htmlForMarkdown( // Add default extensions (Code Highlight, Latex) md = md.use(markdown_it_highlightjs).use(markdownItKatex).use(markdownItClass, classMap); - const mdtext = md.render(markdownText); + let markdownRender = md.render(markdownText); + if (markdownRender.endsWith('\n')) { + // Keep legacy behavior from showdown where the output does not end with \n. + // This is needed because e.g. for quiz questions, we render the markdown in multiple small parts and then concatenate them. + markdownRender = markdownRender.slice(0, -1); + } const purifyParameters = {} as Config; // Prevents sanitizer from deleting id @@ -55,7 +60,7 @@ export function htmlForMarkdown( if (allowedHtmlAttributes) { purifyParameters['ALLOWED_ATTR'] = allowedHtmlAttributes; } - return DOMPurify.sanitize(mdtext, purifyParameters) as string; + return DOMPurify.sanitize(markdownRender, purifyParameters) as string; } export function markdownForHtml(htmlText: string): string { diff --git a/src/test/javascript/spec/component/shared/metis/posting-content/posting-content-part.component.spec.ts b/src/test/javascript/spec/component/shared/metis/posting-content/posting-content-part.component.spec.ts index 81e073bf43a6..424567dd6518 100644 --- a/src/test/javascript/spec/component/shared/metis/posting-content/posting-content-part.component.spec.ts +++ b/src/test/javascript/spec/component/shared/metis/posting-content/posting-content-part.component.spec.ts @@ -94,7 +94,7 @@ describe('PostingContentPartComponent', () => { expect(markdownRenderedTexts).toHaveLength(2); // check that the paragraph right before the reference and the paragraph right after have the class `inline-paragraph` expect(markdownRenderedTexts![0].innerHTML).toInclude('

Be aware

'); - expect(markdownRenderedTexts![0].innerHTML).toInclude('

I want to reference the following Post

'); // last paragraph before reference + expect(markdownRenderedTexts![0].innerHTML).toInclude('

I want to reference the following Post

'); // last paragraph before reference expect(markdownRenderedTexts![1].innerHTML).toInclude('

in my content,

'); // first paragraph after reference expect(markdownRenderedTexts![1].innerHTML).toInclude('

does it actually work?

'); From 6c2457eb23c0c46db171921aba4bdae75bafa462 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 18:05:54 +0200 Subject: [PATCH 09/21] fix problem statement test --- src/test/javascript/spec/helpers/sample/problemStatement.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/javascript/spec/helpers/sample/problemStatement.json b/src/test/javascript/spec/helpers/sample/problemStatement.json index 125a7c25d5c9..de6253655fc5 100644 --- a/src/test/javascript/spec/helpers/sample/problemStatement.json +++ b/src/test/javascript/spec/helpers/sample/problemStatement.json @@ -7,8 +7,8 @@ "problemStatementBothFailedRendered": "
    \n
  1. Implement Bubble Sort: artemisApp.editor.testStatusLabels.noResult
    \nImplement the method performSort(List<Date>) in the class BubbleSort. Make sure to follow the Bubble Sort algorithm exactly.
    2. \n
  2. Implement Merge Sort: artemisApp.editor.testStatusLabels.noResult
    \nImplement the method performSort(List<Date>) in the class MergeSort. Make sure to follow the Merge Sort algorithm exactly.
    3. \n
\n", "problemStatementBothFailedHtml": "
    \n
  1. Implement Bubble Sort: artemisApp.editor.testStatusLabels.testFailing
    \nImplement the method performSort(List<Date>) in the class BubbleSort. Make sure to follow the Bubble Sort algorithm exactly.
    2. \n
  2. Implement Merge Sort: artemisApp.editor.testStatusLabels.testPassing
    \nImplement the method performSort(List<Date>) in the class MergeSort. Make sure to follow the Merge Sort algorithm exactly.
    3. \n
\n", "problemStatementBubbleSortFailsRendered": "
    \n
  1. Implement Bubble Sort: artemisApp.editor.testStatusLabels.noResult
    \nImplement the method performSort(List<Date>) in the class BubbleSort. Make sure to follow the Bubble Sort algorithm exactly.
    2. \n
  2. Implement Merge Sort: artemisApp.editor.testStatusLabels.noResult
    \nImplement the method performSort(List<Date>) in the class MergeSort. Make sure to follow the Merge Sort algorithm exactly.
    3. \n
\n", - "problemStatementBubbleSortNotExecutedHtml": "
    \n
  1. Implement Bubble SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":0}]
    \nImplement the method performSort(List<Date>) in the class BubbleSort. Make sure to follow the Bubble Sort algorithm exactly.
    2. \n
  2. Implement Merge SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":1}]
    \nImplement the method performSort(List<Date>) in the class MergeSort. Make sure to follow the Merge Sort algorithm exactly.
    3. \n
", + "problemStatementBubbleSortNotExecutedHtml": "
    \n
  1. Implement Bubble SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":0}]

    \nImplement the method performSort(List<Date>) in the class BubbleSort. Make sure to follow the Bubble Sort algorithm exactly.
    2. \n
  2. Implement Merge SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":1}]

    \nImplement the method performSort(List<Date>) in the class MergeSort. Make sure to follow the Merge Sort algorithm exactly.
    3. \n
", "problemStatementEmptySecondTask": "1. [task][Bubble Sort](1) \n Implement the method. \n 2. [task][Merge Sort]() \n Implement the method.", - "problemStatementEmptySecondTaskNotExecutedHtml": "
    \n
  1. Bubble SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":1}]
    \nImplement the method.
    2. \n
  2. Merge SortartemisApp.editor.testStatusLabels.noTests
    \nImplement the method.
    3. \n
", + "problemStatementEmptySecondTaskNotExecutedHtml": "
    \n
  1. Bubble SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":1}]

    \nImplement the method.
    2. \n
  2. Merge SortartemisApp.editor.testStatusLabels.noTests

    \nImplement the method.
    3. \n
", "problemStatementPlantUMLWithTest": "@startuml\nclass Policy {\n1)>+configure()\n2)>+testWithParenthesis()}\n@enduml" } From 288ba018205f2a30cecab13fab63e150cccb57d3 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 18:09:16 +0200 Subject: [PATCH 10/21] fix text unit test --- .../lecture-unit/text-unit/text-unit.component.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/javascript/spec/component/lecture-unit/text-unit/text-unit.component.spec.ts b/src/test/javascript/spec/component/lecture-unit/text-unit/text-unit.component.spec.ts index dc8ffa8669af..3326a8e91514 100644 --- a/src/test/javascript/spec/component/lecture-unit/text-unit/text-unit.component.spec.ts +++ b/src/test/javascript/spec/component/lecture-unit/text-unit/text-unit.component.spec.ts @@ -27,7 +27,7 @@ describe('TextUnitComponent', () => { visibleToStudents: true, }; - const exampleHtml = '

Sample Markdown

'; + const exampleHtml = '

Sample Markdown

'; beforeEach(async () => { await TestBed.configureTestingModule({ From aa74d4836a31c026ee7d50fab917316fcfc5deca Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 20:16:50 +0200 Subject: [PATCH 11/21] fix task linebreak --- src/main/webapp/app/shared/util/markdown.conversion.util.ts | 2 +- src/test/javascript/spec/helpers/sample/problemStatement.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index 998abb524607..2986ed5f8ace 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -35,7 +35,7 @@ export function htmlForMarkdown( let md = markdownIt({ html: true, linkify: true, - breaks: true, + breaks: false, // Avoid line breaks after tasks }); for (const extension of extensions) { md = md.use(extension); diff --git a/src/test/javascript/spec/helpers/sample/problemStatement.json b/src/test/javascript/spec/helpers/sample/problemStatement.json index de6253655fc5..6a9297f35013 100644 --- a/src/test/javascript/spec/helpers/sample/problemStatement.json +++ b/src/test/javascript/spec/helpers/sample/problemStatement.json @@ -7,8 +7,8 @@ "problemStatementBothFailedRendered": "
    \n
  1. Implement Bubble Sort: artemisApp.editor.testStatusLabels.noResult
    \nImplement the method performSort(List<Date>) in the class BubbleSort. Make sure to follow the Bubble Sort algorithm exactly.
    2. \n
  2. Implement Merge Sort: artemisApp.editor.testStatusLabels.noResult
    \nImplement the method performSort(List<Date>) in the class MergeSort. Make sure to follow the Merge Sort algorithm exactly.
    3. \n
\n", "problemStatementBothFailedHtml": "
    \n
  1. Implement Bubble Sort: artemisApp.editor.testStatusLabels.testFailing
    \nImplement the method performSort(List<Date>) in the class BubbleSort. Make sure to follow the Bubble Sort algorithm exactly.
    2. \n
  2. Implement Merge Sort: artemisApp.editor.testStatusLabels.testPassing
    \nImplement the method performSort(List<Date>) in the class MergeSort. Make sure to follow the Merge Sort algorithm exactly.
    3. \n
\n", "problemStatementBubbleSortFailsRendered": "
    \n
  1. Implement Bubble Sort: artemisApp.editor.testStatusLabels.noResult
    \nImplement the method performSort(List<Date>) in the class BubbleSort. Make sure to follow the Bubble Sort algorithm exactly.
    2. \n
  2. Implement Merge Sort: artemisApp.editor.testStatusLabels.noResult
    \nImplement the method performSort(List<Date>) in the class MergeSort. Make sure to follow the Merge Sort algorithm exactly.
    3. \n
\n", - "problemStatementBubbleSortNotExecutedHtml": "
    \n
  1. Implement Bubble SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":0}]

    \nImplement the method performSort(List<Date>) in the class BubbleSort. Make sure to follow the Bubble Sort algorithm exactly.
    2. \n
  2. Implement Merge SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":1}]

    \nImplement the method performSort(List<Date>) in the class MergeSort. Make sure to follow the Merge Sort algorithm exactly.
    3. \n
", + "problemStatementBubbleSortNotExecutedHtml": "
    \n
  1. Implement Bubble SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":0}]
    \nImplement the method performSort(List<Date>) in the class BubbleSort. Make sure to follow the Bubble Sort algorithm exactly.
    2. \n
  2. Implement Merge SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":1}]
    \nImplement the method performSort(List<Date>) in the class MergeSort. Make sure to follow the Merge Sort algorithm exactly.
    3. \n
", "problemStatementEmptySecondTask": "1. [task][Bubble Sort](1) \n Implement the method. \n 2. [task][Merge Sort]() \n Implement the method.", - "problemStatementEmptySecondTaskNotExecutedHtml": "
    \n
  1. Bubble SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":1}]

    \nImplement the method.
    2. \n
  2. Merge SortartemisApp.editor.testStatusLabels.noTests

    \nImplement the method.
    3. \n
", + "problemStatementEmptySecondTaskNotExecutedHtml": "
    \n
  1. Bubble SortartemisApp.editor.testStatusLabels.totalTestsPassing: [{\"totalTests\":1,\"passedTests\":1}]
    \nImplement the method.
    2. \n
  2. Merge SortartemisApp.editor.testStatusLabels.noTests
    \nImplement the method.
    3. \n
", "problemStatementPlantUMLWithTest": "@startuml\nclass Policy {\n1)>+configure()\n2)>+testWithParenthesis()}\n@enduml" } From 1bb82dca9df1a3697b1237b9b7959fbd0ffe80f2 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 21:33:52 +0200 Subject: [PATCH 12/21] use different markdown library --- angular.json | 1 - package-lock.json | 28 +++++++------------ package.json | 7 +---- src/main/webapp/app/index.d.ts | 5 ---- .../shared/util/markdown.conversion.util.ts | 4 +-- 5 files changed, 13 insertions(+), 32 deletions(-) diff --git a/angular.json b/angular.json index b77e49684482..ba5b46179bf3 100644 --- a/angular.json +++ b/angular.json @@ -45,7 +45,6 @@ "rfdc", "shallowequal", "markdown-it-class", - "@iktakahiro/markdown-it-katex", "smoothscroll-polyfill", "sockjs-client", "use-sync-external-store/shim", diff --git a/package-lock.json b/package-lock.json index 586e4d5b5332..77a03c6b8592 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,7 +30,6 @@ "@fortawesome/fontawesome-svg-core": "6.6.0", "@fortawesome/free-regular-svg-icons": "6.6.0", "@fortawesome/free-solid-svg-icons": "6.6.0", - "@iktakahiro/markdown-it-katex": "4.0.1", "@ls1intum/apollon": "3.3.14", "@ng-bootstrap/ng-bootstrap": "17.0.1", "@ngx-translate/core": "15.0.0", @@ -39,6 +38,7 @@ "@swimlane/ngx-charts": "20.5.0", "@swimlane/ngx-graph": "8.4.0", "@vscode/codicons": "0.0.36", + "@vscode/markdown-it-katex": "1.1.0", "bootstrap": "5.3.3", "compare-versions": "6.1.1", "core-js": "3.38.1", @@ -98,7 +98,6 @@ "@types/markdown-it": "14.1.2", "@types/node": "22.5.5", "@types/papaparse": "5.3.14", - "@types/showdown": "2.0.6", "@types/smoothscroll-polyfill": "0.3.4", "@types/sockjs-client": "1.5.4", "@types/turndown": "5.0.5", @@ -3577,15 +3576,6 @@ "react": "*" } }, - "node_modules/@iktakahiro/markdown-it-katex": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@iktakahiro/markdown-it-katex/-/markdown-it-katex-4.0.1.tgz", - "integrity": "sha512-kGFooO7fIOgY34PSG8ZNVsUlKhhNoqhzW2kq94TNGa8COzh73PO4KsEoPOsQVG1mEAe8tg7GqG0FoVao0aMHaw==", - "license": "MIT", - "dependencies": { - "katex": "^0.12.0" - } - }, "node_modules/@inquirer/checkbox": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-2.5.0.tgz", @@ -6746,13 +6736,6 @@ "@types/send": "*" } }, - "node_modules/@types/showdown": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/showdown/-/showdown-2.0.6.tgz", - "integrity": "sha512-pTvD/0CIeqe4x23+YJWlX2gArHa8G0J0Oh6GKaVXV7TAeickpkkZiNOgFcFcmLQ5lB/K0qBJL1FtRYltBfbGCQ==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/smoothscroll-polyfill": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/@types/smoothscroll-polyfill/-/smoothscroll-polyfill-0.3.4.tgz", @@ -7061,6 +7044,15 @@ "integrity": "sha512-wsNOvNMMJ2BY8rC2N2MNBG7yOowV3ov8KlvUE/AiVUlHKTfWsw3OgAOQduX7h0Un6GssKD3aoTVH+TF3DSQwKQ==", "license": "CC-BY-4.0" }, + "node_modules/@vscode/markdown-it-katex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@vscode/markdown-it-katex/-/markdown-it-katex-1.1.0.tgz", + "integrity": "sha512-9cF2eJpsJOEs2V1cCAoJW/boKz9GQQLvZhNvI030K90z6ZE9lRGc9hDVvKut8zdFO2ObjwylPXXXVYvTdP2O2Q==", + "license": "MIT", + "dependencies": { + "katex": "^0.16.4" + } + }, "node_modules/@webassemblyjs/ast": { "version": "1.12.1", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz", diff --git a/package.json b/package.json index d5bee6f88f76..56061f70dc0e 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,6 @@ "@fortawesome/fontawesome-svg-core": "6.6.0", "@fortawesome/free-regular-svg-icons": "6.6.0", "@fortawesome/free-solid-svg-icons": "6.6.0", - "@iktakahiro/markdown-it-katex": "4.0.1", "@ls1intum/apollon": "3.3.14", "@ng-bootstrap/ng-bootstrap": "17.0.1", "@ngx-translate/core": "15.0.0", @@ -42,6 +41,7 @@ "@swimlane/ngx-charts": "20.5.0", "@swimlane/ngx-graph": "8.4.0", "@vscode/codicons": "0.0.36", + "@vscode/markdown-it-katex": "1.1.0", "bootstrap": "5.3.3", "compare-versions": "6.1.1", "core-js": "3.38.1", @@ -102,13 +102,9 @@ "@typescript-eslint/eslint-plugin": "^8.6.0" }, "jsdom": "25.0.0", - "katex": "0.16.11", "postcss": "8.4.47", "rimraf": "6.0.1", "semver": "7.6.3", - "showdown-katex": { - "showdown": "2.1.0" - }, "tough-cookie": "5.0.0", "vite": "5.4.6", "webpack-dev-middleware": "7.4.2", @@ -136,7 +132,6 @@ "@types/markdown-it": "14.1.2", "@types/node": "22.5.5", "@types/papaparse": "5.3.14", - "@types/showdown": "2.0.6", "@types/smoothscroll-polyfill": "0.3.4", "@types/sockjs-client": "1.5.4", "@types/turndown": "5.0.5", diff --git a/src/main/webapp/app/index.d.ts b/src/main/webapp/app/index.d.ts index ce7cfe74d6b5..41fc280ef9ff 100644 --- a/src/main/webapp/app/index.d.ts +++ b/src/main/webapp/app/index.d.ts @@ -2,8 +2,3 @@ declare module 'markdown-it-class' { const main: (md: MarkdownIt) => void; export = main; } - -declare module '@iktakahiro/markdown-it-katex' { - const main: (md: MarkdownIt) => void; - export = main; -} diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index 2986ed5f8ace..8837212894d9 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -2,7 +2,7 @@ import DOMPurify, { Config } from 'dompurify'; import type { PluginSimple } from 'markdown-it'; import markdownIt from 'markdown-it'; import markdownItClass from 'markdown-it-class'; -import markdownItKatex from '@iktakahiro/markdown-it-katex'; +import markdownItKatex from '@vscode/markdown-it-katex'; import markdown_it_highlightjs from 'markdown-it-highlightjs'; import TurndownService from 'turndown'; @@ -12,6 +12,7 @@ import TurndownService from 'turndown'; const classMap: { [key: string]: string } = { table: 'table', }; + /** * Converts markdown into html (string) and sanitizes it. Does NOT declare it as safe to bypass further security * Note: If possible, please use safeHtmlForMarkdown @@ -43,7 +44,6 @@ export function htmlForMarkdown( // Add default extensions (Code Highlight, Latex) md = md.use(markdown_it_highlightjs).use(markdownItKatex).use(markdownItClass, classMap); - let markdownRender = md.render(markdownText); if (markdownRender.endsWith('\n')) { // Keep legacy behavior from showdown where the output does not end with \n. From 9e6bdf02c12f476599c79a243c86a47a412dc740 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 21:57:21 +0200 Subject: [PATCH 13/21] Add inline migrator plugin --- .../shared/util/markdown.conversion.util.ts | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index 8837212894d9..a0836b257124 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -13,6 +13,30 @@ const classMap: { [key: string]: string } = { table: 'table', }; +// An inline math formula has some other characters before or after the formula and uses $$ as delimiters +const inlineFormularRegex = /(?:.+\$\$[^\$]+\$\$)|(?:\$\$[^\$]+\$\$.+)/g; +const formulaCompatibilityPlugin: PluginSimple = (md) => { + md.core.ruler.before('inline', 'latex-inline-migrator', (state) => { + // markdownItKatex always creates a big block formula if $$ is used as a deliminator + // which is different from the showdown behavior and could break existing exercises. + // So we replace these with $formular$ to make them inline. + state.tokens.forEach((token) => { + if (token.type === 'inline') { + if (token.content.match(inlineFormularRegex) && token.children) { + console.log(token.content); + token.content = token.content.replace(/\$\$/g, '$'); + for (const child of token.children) { + console.log('child', child.content); + if (child.type === 'text') { + child.content = child.content.replace(/\$\$/g, '$'); + } + } + } + } + }); + }); +}; + /** * Converts markdown into html (string) and sanitizes it. Does NOT declare it as safe to bypass further security * Note: If possible, please use safeHtmlForMarkdown @@ -43,7 +67,13 @@ export function htmlForMarkdown( } // Add default extensions (Code Highlight, Latex) - md = md.use(markdown_it_highlightjs).use(markdownItKatex).use(markdownItClass, classMap); + md = md + .use(markdown_it_highlightjs) + .use(formulaCompatibilityPlugin) + .use(markdownItKatex, { + enableMathInlineInHtml: true, + }) + .use(markdownItClass, classMap); let markdownRender = md.render(markdownText); if (markdownRender.endsWith('\n')) { // Keep legacy behavior from showdown where the output does not end with \n. From 6d7c856b4f0369cf965d181a03a32a52d423f929 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 21:59:30 +0200 Subject: [PATCH 14/21] remove console log --- src/main/webapp/app/shared/util/markdown.conversion.util.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index a0836b257124..7fc8e95bbb12 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -23,10 +23,8 @@ const formulaCompatibilityPlugin: PluginSimple = (md) => { state.tokens.forEach((token) => { if (token.type === 'inline') { if (token.content.match(inlineFormularRegex) && token.children) { - console.log(token.content); token.content = token.content.replace(/\$\$/g, '$'); for (const child of token.children) { - console.log('child', child.content); if (child.type === 'text') { child.content = child.content.replace(/\$\$/g, '$'); } From 762bf2e62431eb3e307802f9b34d2f7f2fdb072e Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 22:14:31 +0200 Subject: [PATCH 15/21] fix matrix representation --- .../app/shared/util/markdown.conversion.util.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index 7fc8e95bbb12..b9780918f4ea 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -21,15 +21,10 @@ const formulaCompatibilityPlugin: PluginSimple = (md) => { // which is different from the showdown behavior and could break existing exercises. // So we replace these with $formular$ to make them inline. state.tokens.forEach((token) => { - if (token.type === 'inline') { - if (token.content.match(inlineFormularRegex) && token.children) { - token.content = token.content.replace(/\$\$/g, '$'); - for (const child of token.children) { - if (child.type === 'text') { - child.content = child.content.replace(/\$\$/g, '$'); - } - } - } + if (token.content.match(inlineFormularRegex)) { + token.content = token.content.replace(/\$\$/g, '$'); + } else if (token.content.includes('\\\\begin') || token.content.includes('\\\\end')) { + token.content = token.content.replaceAll('\\\\begin', '\\begin').replaceAll('\\\\end', '\\end'); } }); }); From e924b213ef8b5613756f4f947a029e1c90a32af5 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Sun, 22 Sep 2024 22:20:20 +0200 Subject: [PATCH 16/21] AI review --- src/main/webapp/app/shared/markdown.service.ts | 2 +- .../app/shared/util/markdown.conversion.util.ts | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/webapp/app/shared/markdown.service.ts b/src/main/webapp/app/shared/markdown.service.ts index c7feb49eb4a0..24e2589c616a 100644 --- a/src/main/webapp/app/shared/markdown.service.ts +++ b/src/main/webapp/app/shared/markdown.service.ts @@ -11,7 +11,7 @@ export class ArtemisMarkdownService { * Converts markdown into html, sanitizes it and then declares it as safe to bypass further security. * * @param {string} markdownText the original markdown text - * @param extensions to use for markdown parsing + * @param {PluginSimple[]} extensions to use for markdown parsing * @param {string[]} allowedHtmlTags to allow during sanitization * @param {string[]} allowedHtmlAttributes to allow during sanitization * @returns {string} the resulting html as a SafeHtml object that can be inserted into the angular template diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index b9780918f4ea..a5228a21e5dd 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -14,14 +14,14 @@ const classMap: { [key: string]: string } = { }; // An inline math formula has some other characters before or after the formula and uses $$ as delimiters -const inlineFormularRegex = /(?:.+\$\$[^\$]+\$\$)|(?:\$\$[^\$]+\$\$.+)/g; +const inlineFormulaRegex = /(?:.+\$\$[^\$]+\$\$)|(?:\$\$[^\$]+\$\$.+)/g; const formulaCompatibilityPlugin: PluginSimple = (md) => { md.core.ruler.before('inline', 'latex-inline-migrator', (state) => { // markdownItKatex always creates a big block formula if $$ is used as a deliminator // which is different from the showdown behavior and could break existing exercises. - // So we replace these with $formular$ to make them inline. + // So we replace these with $formula$ to make them inline. state.tokens.forEach((token) => { - if (token.content.match(inlineFormularRegex)) { + if (token.content.match(inlineFormulaRegex)) { token.content = token.content.replace(/\$\$/g, '$'); } else if (token.content.includes('\\\\begin') || token.content.includes('\\\\end')) { token.content = token.content.replaceAll('\\\\begin', '\\begin').replaceAll('\\\\end', '\\end'); @@ -50,18 +50,17 @@ export function htmlForMarkdown( return ''; } - let md = markdownIt({ + const md = markdownIt({ html: true, linkify: true, breaks: false, // Avoid line breaks after tasks }); for (const extension of extensions) { - md = md.use(extension); + md.use(extension); } // Add default extensions (Code Highlight, Latex) - md = md - .use(markdown_it_highlightjs) + md.use(markdown_it_highlightjs) .use(formulaCompatibilityPlugin) .use(markdownItKatex, { enableMathInlineInHtml: true, From 8ff9acba791d4713ba7745b4814087a88e30ac9a Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Mon, 23 Sep 2024 10:21:55 +0200 Subject: [PATCH 17/21] reuse turndown service object --- src/main/webapp/app/shared/util/markdown.conversion.util.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index a5228a21e5dd..1a55f3e3ca40 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -30,6 +30,8 @@ const formulaCompatibilityPlugin: PluginSimple = (md) => { }); }; +const turndownService = new TurndownService(); + /** * Converts markdown into html (string) and sanitizes it. Does NOT declare it as safe to bypass further security * Note: If possible, please use safeHtmlForMarkdown @@ -86,6 +88,5 @@ export function htmlForMarkdown( } export function markdownForHtml(htmlText: string): string { - const turndownService = new TurndownService(); return turndownService.turndown(htmlText); } From cf16fb8ba7a4edbc1066e167b7e97e94cadce87b Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Mon, 23 Sep 2024 11:27:50 +0200 Subject: [PATCH 18/21] refactor plugin, add client test --- ...rogramming-exercise-plant-uml.extension.ts | 4 +-- .../programming-exercise-task.extension.ts | 4 +-- ...ion.ts => ArtemisTextReplacementPlugin.ts} | 4 +-- .../shared/util/markdown.conversion.util.ts | 35 +++++++++++-------- .../spec/service/markdown.service.spec.ts | 26 ++++++++++++++ 5 files changed, 52 insertions(+), 21 deletions(-) rename src/main/webapp/app/shared/markdown-editor/extensions/{ArtemisTextReplacementExtension.ts => ArtemisTextReplacementPlugin.ts} (83%) diff --git a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts index 9aab41e422cc..6bd987ac42e9 100644 --- a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts +++ b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts @@ -1,6 +1,6 @@ import { Injectable } from '@angular/core'; import { ProgrammingExerciseTestCase } from 'app/entities/programming/programming-exercise-test-case.model'; -import { ArtemisTextReplacementExtension } from 'app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension'; +import { ArtemisTextReplacementPlugin } from 'app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin'; import { escapeStringForUseInRegex } from 'app/shared/util/global.utils'; import { Subject } from 'rxjs'; import { tap } from 'rxjs/operators'; @@ -13,7 +13,7 @@ import DOMPurify from 'dompurify'; const testsColorRegex = /testsColor\((\s*[^()\s]+(\([^()]*\))?)\)/g; @Injectable({ providedIn: 'root' }) -export class ProgrammingExercisePlantUmlExtensionWrapper extends ArtemisTextReplacementExtension { +export class ProgrammingExercisePlantUmlExtensionWrapper extends ArtemisTextReplacementPlugin { private latestResult?: Result; private testCases?: ProgrammingExerciseTestCase[]; private injectableElementsFoundSubject = new Subject<() => void>(); diff --git a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts index d3157bf30d50..c6cbc4f1a2a2 100644 --- a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts +++ b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-task.extension.ts @@ -1,6 +1,6 @@ import { Injectable, ViewContainerRef } from '@angular/core'; import { TaskArrayWithExercise } from 'app/exercises/programming/shared/instructions-render/task/programming-exercise-task.model'; -import { ArtemisTextReplacementExtension } from 'app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension'; +import { ArtemisTextReplacementPlugin } from 'app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin'; import { Observable, Subject } from 'rxjs'; /** @@ -17,7 +17,7 @@ import { Observable, Subject } from 'rxjs'; export const taskRegex = /\[task]\[([^[\]]+)]\(((?:[^(),]+(?:\([^()]*\)[^(),]*)?(?:,[^(),]+(?:\([^()]*\)[^(),]*)?)*)?)\)/g; @Injectable({ providedIn: 'root' }) -export class ProgrammingExerciseTaskExtensionWrapper extends ArtemisTextReplacementExtension { +export class ProgrammingExerciseTaskExtensionWrapper extends ArtemisTextReplacementPlugin { // We don't have a provider for ViewContainerRef, so we pass it from ProgrammingExerciseInstructionComponent viewContainerRef: ViewContainerRef; diff --git a/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension.ts b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts similarity index 83% rename from src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension.ts rename to src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts index 51b8a273d2e7..00ba0f764e48 100644 --- a/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementExtension.ts +++ b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts @@ -2,9 +2,9 @@ import type MarkdownIt from 'markdown-it'; import type { PluginSimple } from 'markdown-it'; /** - * Markdown-It extension that allows replacing text in the raw markdown before tokenizing. + * Markdown-It plugin that allows replacing text in the raw markdown before tokenizing. */ -export abstract class ArtemisTextReplacementExtension { +export abstract class ArtemisTextReplacementPlugin { getExtension(): PluginSimple { return (md: MarkdownIt): void => { // Override the `render` method to process the raw Markdown text before tokenizing diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index 1a55f3e3ca40..bdf79b1a1744 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -1,3 +1,4 @@ +import { ArtemisTextReplacementPlugin } from 'app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin'; import DOMPurify, { Config } from 'dompurify'; import type { PluginSimple } from 'markdown-it'; import markdownIt from 'markdown-it'; @@ -15,20 +16,24 @@ const classMap: { [key: string]: string } = { // An inline math formula has some other characters before or after the formula and uses $$ as delimiters const inlineFormulaRegex = /(?:.+\$\$[^\$]+\$\$)|(?:\$\$[^\$]+\$\$.+)/g; -const formulaCompatibilityPlugin: PluginSimple = (md) => { - md.core.ruler.before('inline', 'latex-inline-migrator', (state) => { - // markdownItKatex always creates a big block formula if $$ is used as a deliminator - // which is different from the showdown behavior and could break existing exercises. - // So we replace these with $formula$ to make them inline. - state.tokens.forEach((token) => { - if (token.content.match(inlineFormulaRegex)) { - token.content = token.content.replace(/\$\$/g, '$'); - } else if (token.content.includes('\\\\begin') || token.content.includes('\\\\end')) { - token.content = token.content.replaceAll('\\\\begin', '\\begin').replaceAll('\\\\end', '\\end'); - } - }); - }); -}; + +class FormularCompatibilityPlugin extends ArtemisTextReplacementPlugin { + replaceText(text: string): string { + return text + .split('\n') + .map((line) => { + if (line.match(inlineFormulaRegex)) { + line = line.replace(/\$\$/g, '$'); + } + if (line.includes('\\\\begin') || line.includes('\\\\end')) { + line = line.replaceAll('\\\\begin', '\\begin').replaceAll('\\\\end', '\\end'); + } + return line; + }) + .join('\n'); + } +} +const formulaCompatibilityPlugin = new FormularCompatibilityPlugin(); const turndownService = new TurndownService(); @@ -63,7 +68,7 @@ export function htmlForMarkdown( // Add default extensions (Code Highlight, Latex) md.use(markdown_it_highlightjs) - .use(formulaCompatibilityPlugin) + .use(formulaCompatibilityPlugin.getExtension()) .use(markdownItKatex, { enableMathInlineInHtml: true, }) diff --git a/src/test/javascript/spec/service/markdown.service.spec.ts b/src/test/javascript/spec/service/markdown.service.spec.ts index 6473a2b6405d..4756e51d47f4 100644 --- a/src/test/javascript/spec/service/markdown.service.spec.ts +++ b/src/test/javascript/spec/service/markdown.service.spec.ts @@ -108,4 +108,30 @@ describe('Markdown Service', () => { const safeMarkdownWithoutExtras = htmlForMarkdown(markdownString, [], [], []); expect(safeMarkdownWithoutExtras).toBe('Will this render blue?'); }); + + describe('formulaCompatibilityPlugin', () => { + it.each(['This is a formula $$E=mc^2$$ in text.', '$$a_1$$ formula at front', 'formula at back $$a_2$$'])('converts block formulas to inline formulas', (input) => { + const result = htmlForMarkdown(input); + expect(result).toContain(''); + expect(result).not.toContain('class="katex-block"'); + }); + + it('does not convert block formulas without surrounding text', () => { + const result = htmlForMarkdown('$$E=mc^2$$'); + expect(result).toContain('class="katex-block"'); + expect(result).toContain('display="block"'); + }); + + it('converts double-backslash LaTeX begin and end tags', () => { + const result = htmlForMarkdown('Here is some LaTeX: $$\\\\begin{equation}a^2 + b^2 = c^2\\\\end{equation}$$\n'); + expect(result).toContain(''); + expect(result).toContain('class="katex-html"'); + }); + + it('handles multiple formulas in the same text', () => { + const result = htmlForMarkdown('First formula $$a^2 + b^2 = c^2$$ and second formula $$E=mc^2$$.'); + const formulaCount = (result.match(/class="katex"/g) || []).length; + expect(formulaCount).toBe(2); + }); + }); }); From faa3137b46045fa8a83b5e70148e7282e3b61957 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Mon, 23 Sep 2024 11:36:49 +0200 Subject: [PATCH 19/21] Update src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../extensions/ArtemisTextReplacementPlugin.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts index 00ba0f764e48..94ce0849f97a 100644 --- a/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts +++ b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts @@ -18,5 +18,10 @@ export abstract class ArtemisTextReplacementPlugin { }; } + /** + * Performs text replacement on the raw markdown before parsing. + * @param text The raw markdown text. + * @returns The modified markdown text after replacements. + */ abstract replaceText(text: string): string; } From c4231421f2ca58323809d05ebd04c43bd742ce76 Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Mon, 23 Sep 2024 11:42:00 +0200 Subject: [PATCH 20/21] AI review --- .../programming-exercise-plant-uml.extension.ts | 2 +- .../extensions/ArtemisTextReplacementPlugin.ts | 10 +++------- .../webapp/app/shared/util/markdown.conversion.util.ts | 8 ++++---- .../javascript/spec/service/markdown.service.spec.ts | 2 ++ 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts index 6bd987ac42e9..886520d929f4 100644 --- a/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts +++ b/src/main/webapp/app/exercises/programming/shared/instructions-render/extensions/programming-exercise-plant-uml.extension.ts @@ -99,7 +99,7 @@ export class ProgrammingExercisePlantUmlExtensionWrapper extends ArtemisTextRepl // before we send the plantUml to the server for rendering, we need to inject the current test status so that the colors can be adapted // (green == implemented, red == not yet implemented, grey == unknown) const plantUmlsValidated = plantUmlsIndexed.map((plantUmlIndexed: { plantUmlId: number; plantUml: string }) => { - plantUmlIndexed.plantUml = plantUmlIndexed.plantUml.replace(testsColorRegex, (match: any, capture: string) => { + plantUmlIndexed.plantUml = plantUmlIndexed.plantUml.replace(testsColorRegex, (match: string, capture: string) => { const tests = this.programmingExerciseInstructionService.convertTestListToIds(capture, this.testCases); const { testCaseState } = this.programmingExerciseInstructionService.testStatusForTask(tests, this.latestResult); switch (testCaseState) { diff --git a/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts index 00ba0f764e48..1b248ea76413 100644 --- a/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts +++ b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts @@ -7,14 +7,10 @@ import type { PluginSimple } from 'markdown-it'; export abstract class ArtemisTextReplacementPlugin { getExtension(): PluginSimple { return (md: MarkdownIt): void => { - // Override the `render` method to process the raw Markdown text before tokenizing - const originalRender = md.render.bind(md); - md.render = (markdownText: string, ...args) => { + md.core.ruler.before('normalize', 'artemis_text_replacement', (state) => { // Perform the replacement on the raw markdown text - const modifiedText = this.replaceText(markdownText); - // Call the original render method with the modified text - return originalRender(modifiedText, ...args); - }; + state.src = this.replaceText(state.src); + }); }; } diff --git a/src/main/webapp/app/shared/util/markdown.conversion.util.ts b/src/main/webapp/app/shared/util/markdown.conversion.util.ts index bdf79b1a1744..d5166b70d690 100644 --- a/src/main/webapp/app/shared/util/markdown.conversion.util.ts +++ b/src/main/webapp/app/shared/util/markdown.conversion.util.ts @@ -4,7 +4,7 @@ import type { PluginSimple } from 'markdown-it'; import markdownIt from 'markdown-it'; import markdownItClass from 'markdown-it-class'; import markdownItKatex from '@vscode/markdown-it-katex'; -import markdown_it_highlightjs from 'markdown-it-highlightjs'; +import markdownItHighlightjs from 'markdown-it-highlightjs'; import TurndownService from 'turndown'; /** @@ -17,7 +17,7 @@ const classMap: { [key: string]: string } = { // An inline math formula has some other characters before or after the formula and uses $$ as delimiters const inlineFormulaRegex = /(?:.+\$\$[^\$]+\$\$)|(?:\$\$[^\$]+\$\$.+)/g; -class FormularCompatibilityPlugin extends ArtemisTextReplacementPlugin { +class FormulaCompatibilityPlugin extends ArtemisTextReplacementPlugin { replaceText(text: string): string { return text .split('\n') @@ -33,7 +33,7 @@ class FormularCompatibilityPlugin extends ArtemisTextReplacementPlugin { .join('\n'); } } -const formulaCompatibilityPlugin = new FormularCompatibilityPlugin(); +const formulaCompatibilityPlugin = new FormulaCompatibilityPlugin(); const turndownService = new TurndownService(); @@ -67,7 +67,7 @@ export function htmlForMarkdown( } // Add default extensions (Code Highlight, Latex) - md.use(markdown_it_highlightjs) + md.use(markdownItHighlightjs) .use(formulaCompatibilityPlugin.getExtension()) .use(markdownItKatex, { enableMathInlineInHtml: true, diff --git a/src/test/javascript/spec/service/markdown.service.spec.ts b/src/test/javascript/spec/service/markdown.service.spec.ts index 4756e51d47f4..d33f0113eea9 100644 --- a/src/test/javascript/spec/service/markdown.service.spec.ts +++ b/src/test/javascript/spec/service/markdown.service.spec.ts @@ -132,6 +132,8 @@ describe('Markdown Service', () => { const result = htmlForMarkdown('First formula $$a^2 + b^2 = c^2$$ and second formula $$E=mc^2$$.'); const formulaCount = (result.match(/class="katex"/g) || []).length; expect(formulaCount).toBe(2); + expect(result).not.toContain('class="katex-block"'); + expect(result).not.toContain('display="block"'); }); }); }); From 2449de5471e81126f4a15759fa034a4277adbcbb Mon Sep 17 00:00:00 2001 From: Lucas Welscher Date: Mon, 23 Sep 2024 12:19:22 +0200 Subject: [PATCH 21/21] Add comment --- .../markdown-editor/extensions/ArtemisTextReplacementPlugin.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts index cd08c7d77c5f..0a913251fc6d 100644 --- a/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts +++ b/src/main/webapp/app/shared/markdown-editor/extensions/ArtemisTextReplacementPlugin.ts @@ -3,6 +3,7 @@ import type { PluginSimple } from 'markdown-it'; /** * Markdown-It plugin that allows replacing text in the raw markdown before tokenizing. + * See more about Markdown-It plugins here: https://github.com/markdown-it/markdown-it/tree/master/docs */ export abstract class ArtemisTextReplacementPlugin { getExtension(): PluginSimple {