From 6b9a88eba530a025b9c3cd49aee88657cb3f8e0a Mon Sep 17 00:00:00 2001 From: ethan92429 Date: Thu, 21 Sep 2023 21:40:35 -0400 Subject: [PATCH 1/6] Accept map substitutions --- .gitignore | 3 ++- sphinx_ext_substitution/get_replacements.py | 10 +++++++++- sphinx_ext_substitution/test_basic.py | 3 +++ testdata/proj/index.rst | 2 ++ testdata/substitutions/one-yaml/subs.yaml | 4 ++++ 5 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index ebbe132..c306f52 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /testdata/proj/_build* -/sphinx_ext_substitution.egg-info/ \ No newline at end of file +/sphinx_ext_substitution.egg-info/ +.idea \ No newline at end of file diff --git a/sphinx_ext_substitution/get_replacements.py b/sphinx_ext_substitution/get_replacements.py index 75cf879..810dcd5 100644 --- a/sphinx_ext_substitution/get_replacements.py +++ b/sphinx_ext_substitution/get_replacements.py @@ -29,9 +29,17 @@ def _load_yaml(fname, substitutions): if yaml is None: raise RuntimeError("The yaml module is needed (python-yaml) to get definitions from yaml files.") data = yaml.load(open(fname), Loader=yaml.SafeLoader) + _load_substitutions_map(data, substitutions) + + +def _load_substitutions_map(data, substitutions, prefix=""): for key, value in data.items(): if key not in substitutions: - substitutions[key] = value.strip() + if isinstance(value, str): + substitutions[f"{prefix}{key}"] = value.strip() + elif isinstance(value, dict): + _load_substitutions_map(value, substitutions, f"{prefix}{key}.") + def load_substitutions(config): """Load substitutions from disk. Cache results to SUBSTITUTIONS. diff --git a/sphinx_ext_substitution/test_basic.py b/sphinx_ext_substitution/test_basic.py index 6bb8ed7..b13851d 100644 --- a/sphinx_ext_substitution/test_basic.py +++ b/sphinx_ext_substitution/test_basic.py @@ -33,6 +33,9 @@ def doc1_original(): return doc(build='_build-original', opts="-D substitute_mode=original") +def test_map_id(doc1_default): + index = doc1_default['index'] + assert "A13-id-substitute" in index def test_role(doc1_default): index = doc1_default['index'] diff --git a/testdata/proj/index.rst b/testdata/proj/index.rst index 0b31256..7edf13b 100644 --- a/testdata/proj/index.rst +++ b/testdata/proj/index.rst @@ -31,6 +31,8 @@ No replacement (preformatted): :sub:`A7-id: \`\`A7-original\`\`` Replacement (preformatted): :sub:`A8-id: \`\`A8-original\`\`` +Map Replacement :sub:`A12-id.A13-id:\`\`A13-original\`\`` + Directive with no replacement ----------------------------- diff --git a/testdata/substitutions/one-yaml/subs.yaml b/testdata/substitutions/one-yaml/subs.yaml index 1c191fb..4dd7d51 100644 --- a/testdata/substitutions/one-yaml/subs.yaml +++ b/testdata/substitutions/one-yaml/subs.yaml @@ -6,3 +6,7 @@ A11-id: | A11.1-substitute *A11.2-substitute* + +A12-id: + A13-id: A13-id-substitute + A14-id: A14-id-substitute \ No newline at end of file From a2ba3d5d3e5083a73a603a11c66729b0d358912c Mon Sep 17 00:00:00 2001 From: ethan92429 Date: Tue, 26 Sep 2023 17:36:54 -0400 Subject: [PATCH 2/6] Find substring in substitution map --- sphinx_ext_substitution/substitution.py | 10 ++++++---- testdata/proj/index.rst | 10 ++++++++++ testdata/substitutions/one-yaml/subs.yaml | 7 ++++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/sphinx_ext_substitution/substitution.py b/sphinx_ext_substitution/substitution.py index 5e48ca2..8d5dcfe 100644 --- a/sphinx_ext_substitution/substitution.py +++ b/sphinx_ext_substitution/substitution.py @@ -66,10 +66,12 @@ def sub_role(name, rawtext, text, lineno, inliner, original = original.replace('\x00`', '`') # Find the replacement value, don't use it for anything yet. - if id_ in subs: - replacement = subs[id_] - else: - replacement = None + replacement = None + for potential_id, value in subs.items(): + if id_ in potential_id: + if replacement is not None: + raise UserWarning('Two potential replacements found.') + replacement = value if replacement: replacement = replacement.replace('\x00`', '`') diff --git a/testdata/proj/index.rst b/testdata/proj/index.rst index 7edf13b..2338c18 100644 --- a/testdata/proj/index.rst +++ b/testdata/proj/index.rst @@ -33,6 +33,16 @@ Replacement (preformatted): :sub:`A8-id: \`\`A8-original\`\`` Map Replacement :sub:`A12-id.A13-id:\`\`A13-original\`\`` +Map Replacement :sub:`plant.lfom.rowN:\`\`rown-og\`\`` + +Map Replacement :sub:`lfom.rowN:\`\`rown-og\`\`` + +Map Replacement :sub:`rowN:\`\`rown-og\`\`` + +Map Replacement :sub:`rowB:\`\`rown-og\`\`` + +Map Replacement :sub:`rowb:\`\`rown-og\`\`` + Directive with no replacement ----------------------------- diff --git a/testdata/substitutions/one-yaml/subs.yaml b/testdata/substitutions/one-yaml/subs.yaml index 4dd7d51..483611f 100644 --- a/testdata/substitutions/one-yaml/subs.yaml +++ b/testdata/substitutions/one-yaml/subs.yaml @@ -9,4 +9,9 @@ A11-id: | A12-id: A13-id: A13-id-substitute - A14-id: A14-id-substitute \ No newline at end of file + A14-id: A14-id-substitute + +plant: + lfom: + rowN: '5' + rowB: '5 cm' \ No newline at end of file From 4d9753c9d626a61f9551e7ace2dfb444bd13a5cf Mon Sep 17 00:00:00 2001 From: ethan92429 Date: Wed, 27 Sep 2023 08:22:59 -0400 Subject: [PATCH 3/6] Update the Readme with new features. --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 2b65c0c..b6e1331 100644 --- a/README.rst +++ b/README.rst @@ -23,6 +23,10 @@ Features include: - Both role (inline) and directive (paragraph level) support. - Theoretical Markdown support via `myst_parser `__. +- Support nested mappings within yaml using `.` to drill into keys. +- Partial key lookup support. You only need to specify some substring + within the key to get the substitution. If it matches more than one, + an error will be thrown. Demo From 64e1852b44ea328438e1bf9225fff3cd5688e82e Mon Sep 17 00:00:00 2001 From: ethan92429 Date: Thu, 28 Sep 2023 18:04:00 -0400 Subject: [PATCH 4/6] Add jsonPath (WIP) --- .gitignore | 3 +-- README.rst | 2 ++ requirements.txt | 1 + sphinx_ext_substitution/get_replacements.py | 12 ++++++--- sphinx_ext_substitution/substitution.py | 29 ++++++++++++++++----- sphinx_ext_substitution/test_basic.py | 26 ++++++++++++++++++ testdata/proj/index.rst | 12 ++++----- 7 files changed, 67 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index c306f52..ebbe132 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ /testdata/proj/_build* -/sphinx_ext_substitution.egg-info/ -.idea \ No newline at end of file +/sphinx_ext_substitution.egg-info/ \ No newline at end of file diff --git a/README.rst b/README.rst index b6e1331..8daa13b 100644 --- a/README.rst +++ b/README.rst @@ -197,6 +197,8 @@ Currently there are two Sphinx variables defined: need for both, but it provides more flexible configuration for integration to your build system. +* If ``partial_match_substitution`` set to true, then only a partial + string match to the key will result in a substitution. Development and maintenance diff --git a/requirements.txt b/requirements.txt index 789b15e..b4963d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ Sphinx docutils!=0.15 PyYAML +jsonpath-ng diff --git a/sphinx_ext_substitution/get_replacements.py b/sphinx_ext_substitution/get_replacements.py index 810dcd5..625bf48 100644 --- a/sphinx_ext_substitution/get_replacements.py +++ b/sphinx_ext_substitution/get_replacements.py @@ -29,16 +29,20 @@ def _load_yaml(fname, substitutions): if yaml is None: raise RuntimeError("The yaml module is needed (python-yaml) to get definitions from yaml files.") data = yaml.load(open(fname), Loader=yaml.SafeLoader) - _load_substitutions_map(data, substitutions) + _strip_whitespace_recursively(data, substitutions) + print("substitutions") + print(substitutions) -def _load_substitutions_map(data, substitutions, prefix=""): +def _strip_whitespace_recursively(data, substitutions): + """Save this dict's keys as substitutions, recurse with prefix into sub-dicts.""" for key, value in data.items(): if key not in substitutions: if isinstance(value, str): - substitutions[f"{prefix}{key}"] = value.strip() + substitutions[key] = value.strip() elif isinstance(value, dict): - _load_substitutions_map(value, substitutions, f"{prefix}{key}.") + substitutions[key] = _strip_whitespace_recursively(value, data[key]) + return substitutions def load_substitutions(config): diff --git a/sphinx_ext_substitution/substitution.py b/sphinx_ext_substitution/substitution.py index 8d5dcfe..3a2353b 100644 --- a/sphinx_ext_substitution/substitution.py +++ b/sphinx_ext_substitution/substitution.py @@ -10,6 +10,8 @@ from sphinx.locale import _ import sphinx.util.nodes +from jsonpath_ng import jsonpath, parse + from .get_replacements import get_substitutions class sub(nodes.Admonition, nodes.Element): @@ -66,12 +68,11 @@ def sub_role(name, rawtext, text, lineno, inliner, original = original.replace('\x00`', '`') # Find the replacement value, don't use it for anything yet. - replacement = None - for potential_id, value in subs.items(): - if id_ in potential_id: - if replacement is not None: - raise UserWarning('Two potential replacements found.') - replacement = value + if id_ in subs: + replacement = subs[id_] + else: + replacement = _jsonpath_lookup(id_, subs) + if replacement: replacement = replacement.replace('\x00`', '`') @@ -124,6 +125,20 @@ def sub_role(name, rawtext, text, lineno, inliner, return content, messages +def _jsonpath_lookup(id_, subs): + try: + expression = parse(id_) + except: + return None + matches = [match.value for match in expression.find(subs)] + if len(matches) == 0: + replacement = None + elif all([m == matches[0] for m in matches]): + replacement = matches[0] + else: + raise UserWarning(f'Different replacements found for {id_}... matches found: {matches}') + return replacement + class SubDirective(Directive): required_arguments = 1 @@ -144,6 +159,8 @@ def run(self): # Get the replacement value, don't use it yet if id_ in subs: replacement = subs[id_] + if not isinstance(replacement, str): + raise UserWarning(f"Yields a non-string substitution for {id_}... resulted in {replacement}") replacement = statemachine.StringList( replacement.splitlines(), source='file') else: diff --git a/sphinx_ext_substitution/test_basic.py b/sphinx_ext_substitution/test_basic.py index b13851d..d19e47f 100644 --- a/sphinx_ext_substitution/test_basic.py +++ b/sphinx_ext_substitution/test_basic.py @@ -32,6 +32,32 @@ def doc1_original(): """Document with substitute_mode=original""" return doc(build='_build-original', opts="-D substitute_mode=original") +@pytest.fixture(scope='session') +def doc1_original(): + """Document with partial_match_substitution=true""" + return doc(build='_build-original', opts="-D partial_match_substitution=true") + +def testFindJsonPath(): + from jsonpath_ng import jsonpath, parse + # Sample JSON data + data = { + "name": "John", + "address": { + "city": "New York", + "state": { + "code": "NY", + "name": "New York" + } + } + } + + # Compile the JSONPath expression + expression = parse("$..code") + + # Use the expression to find matches in your data + matches = [match.value for match in expression.find(data)] + + print(matches) # Output: ['NY'] def test_map_id(doc1_default): index = doc1_default['index'] diff --git a/testdata/proj/index.rst b/testdata/proj/index.rst index 2338c18..479e257 100644 --- a/testdata/proj/index.rst +++ b/testdata/proj/index.rst @@ -31,15 +31,15 @@ No replacement (preformatted): :sub:`A7-id: \`\`A7-original\`\`` Replacement (preformatted): :sub:`A8-id: \`\`A8-original\`\`` -Map Replacement :sub:`A12-id.A13-id:\`\`A13-original\`\`` +Map Replacement :sub:`$.A12-id.A13-id:\`\`A13-original\`\`` -Map Replacement :sub:`plant.lfom.rowN:\`\`rown-og\`\`` +Map Replacement :sub:`$.plant.lfom.rowN:\`\`rown-og\`\`` -Map Replacement :sub:`lfom.rowN:\`\`rown-og\`\`` +Map Replacement :sub:`$..lfom.rowN:\`\`rown-og\`\`` -Map Replacement :sub:`rowN:\`\`rown-og\`\`` +Map Replacement :sub:`$..rowN:\`\`rown-og\`\`` -Map Replacement :sub:`rowB:\`\`rown-og\`\`` +Map Replacement :sub:`$..rowB:\`\`rown-og\`\`` Map Replacement :sub:`rowb:\`\`rown-og\`\`` @@ -64,4 +64,4 @@ Directive with replacement Directive with no content ------------------------- -.. sub:: A12-id +.. sub:: $.A12-id.A13-id From 9ae873b116999eed2582ee980b51454041b66619 Mon Sep 17 00:00:00 2001 From: Richard Darst Date: Fri, 20 Oct 2023 01:07:55 +0300 Subject: [PATCH 5/6] sphinx_ext_substitution/test_basic: Rename fixture - This overlapped with doc1_original causing problems. --- sphinx_ext_substitution/test_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx_ext_substitution/test_basic.py b/sphinx_ext_substitution/test_basic.py index d19e47f..b93c716 100644 --- a/sphinx_ext_substitution/test_basic.py +++ b/sphinx_ext_substitution/test_basic.py @@ -33,9 +33,9 @@ def doc1_original(): return doc(build='_build-original', opts="-D substitute_mode=original") @pytest.fixture(scope='session') -def doc1_original(): +def doc1_partial_match(): """Document with partial_match_substitution=true""" - return doc(build='_build-original', opts="-D partial_match_substitution=true") + return doc(build='_build-partial-match', opts="-D partial_match_substitution=true") def testFindJsonPath(): from jsonpath_ng import jsonpath, parse From 09fe6ae25bd826170240b0706058304066031b01 Mon Sep 17 00:00:00 2001 From: Richard Darst Date: Fri, 20 Oct 2023 01:10:17 +0300 Subject: [PATCH 6/6] README: Comment out partial_match_substitution (not implemented yet) --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 8daa13b..b4ca3c3 100644 --- a/README.rst +++ b/README.rst @@ -197,8 +197,9 @@ Currently there are two Sphinx variables defined: need for both, but it provides more flexible configuration for integration to your build system. -* If ``partial_match_substitution`` set to true, then only a partial - string match to the key will result in a substitution. +.. + * If ``partial_match_substitution`` set to true, then only a partial + string match to the key will result in a substitution. Development and maintenance