Merge pull request #859 from NASA-PDS/issue_832

Fixes bugs with erroneous messages from output logs, duplicate records in collections, and handling of `high_instrument_saturation` special constants

src/main/java/gov/nasa/pds/tools/label/LabelValidator.java

@@ -652,7 +652,7 @@ public synchronized Document parseAndValidate(ProblemHandler handler, URL url)
 
         SAXSource saxSource = new SAXSource(Utility.getInputSourceByURL(url));
         saxSource.setSystemId(url.toString());
-        TreeInfo docInfo = LabelParser.parse(saxSource);
+        TreeInfo docInfo = LabelParser.parse(saxSource, false);
         for (DocumentValidator dv : documentValidators) {
           dv.validate(handler, docInfo.getRootNode());
         }

src/main/java/gov/nasa/pds/tools/util/LabelParser.java

@@ -13,25 +13,35 @@
 // $Id$
 package gov.nasa.pds.tools.util;
 
+import java.util.function.Function;
 import javax.xml.transform.Source;
 import javax.xml.transform.TransformerException;
 import net.sf.saxon.Configuration;
+import net.sf.saxon.lib.ErrorReporter;
 import net.sf.saxon.lib.ParseOptions;
 import net.sf.saxon.om.TreeInfo;
+import net.sf.saxon.s9api.XmlProcessingError;
 import net.sf.saxon.xpath.XPathEvaluator;
 
-public class LabelParser {
-
+public class LabelParser implements ErrorReporter {
+  private static class Factory implements Function<Configuration,ErrorReporter> {
+    final ErrorReporter ignorer = new LabelParser();
+    @Override
+    public ErrorReporter apply(Configuration t) {
+      return ignorer;
+    }
+  }
   /**
    * Parses a label.
    * 
    * @param source the source to parse.
    * @return a DocumentInfo object.
    * @throws TransformerException
    */
-  public static TreeInfo parse(Source source) throws TransformerException {
+  public static TreeInfo parse(Source source, boolean ignoreErrors) throws TransformerException {
     XPathEvaluator xpath = new XPathEvaluator();
     Configuration configuration = xpath.getConfiguration();
+    if (ignoreErrors) configuration.setErrorReporterFactory(new Factory());
     configuration.setLineNumbering(true);
     configuration.setXIncludeAware(Utility.supportXincludes());
     ParseOptions options = new ParseOptions();
@@ -40,4 +50,8 @@ public static TreeInfo parse(Source source) throws TransformerException {
     options.withXIncludeAware(Utility.supportXincludes());
     return configuration.buildDocumentTree(source, options);
   }
+  @Override
+  public void report(XmlProcessingError error) {
+    // do nothing
+  }
 }

src/main/java/gov/nasa/pds/tools/validate/TargetExaminer.java

@@ -148,14 +148,11 @@ private static boolean tagMatches(URL url, String tagCheck, boolean ignoreErrors
     // xmlns:pds="http://pds.nasa.gov/pds4/pds/v1"
     boolean tagMatchedFlag = false;
 
-    if (!url.getPath().endsWith(".xml")) {
-      tagMatchedFlag = false;
-    }
     try {
       InputSource source = Utility.getInputSourceByURL(url);
       SAXSource saxSource = new SAXSource(source);
       saxSource.setSystemId(url.toString());
-      TreeInfo docInfo = LabelParser.parse(saxSource); // Parses a label.
+      TreeInfo docInfo = LabelParser.parse(saxSource, ignoreErrors); // Parses a label.
       LOG.debug("tagMatches:docInfo {},{}", docInfo, docInfo.getClass());
       List<TinyNodeImpl> xmlModels = new ArrayList<>();
       try {
@@ -209,7 +206,7 @@ public static ArrayList<String> getTargetContent(URL url, String nodeCheck, Stri
       InputSource source = Utility.getInputSourceByURL(url);
       SAXSource saxSource = new SAXSource(source);
       saxSource.setSystemId(url.toString());
-      TreeInfo docInfo = LabelParser.parse(saxSource); // Parses a label.
+      TreeInfo docInfo = LabelParser.parse(saxSource, false); // Parses a label.
       LOG.debug("getTargetContent:docInfo {},{}", docInfo, docInfo.getClass());
       List<TinyNodeImpl> xmlModels = new ArrayList<>();
       try {
@@ -254,7 +251,7 @@ public static boolean isTargetALabel(URL url) {
       InputSource source = Utility.getInputSourceByURL(url);
       SAXSource saxSource = new SAXSource(source);
       saxSource.setSystemId(url.toString());
-      TreeInfo docInfo = LabelParser.parse(saxSource); // Parses a label.
+      TreeInfo docInfo = LabelParser.parse(saxSource, true); // Parses a label.
       @SuppressWarnings("unused")
       List<TinyNodeImpl> xmlModels = new ArrayList<>();
       XMLExtractor extractor = new XMLExtractor(docInfo.getRootNode());

src/main/java/gov/nasa/pds/tools/validate/content/array/ArrayContentValidator.java

@@ -343,7 +343,9 @@ private static boolean sameContent (Number number, String constant_repr) {
     if (number instanceof Integer) number = BigInteger.valueOf(number.intValue());
     if (number instanceof Long) number = BigInteger.valueOf(number.longValue());
     if (number instanceof Short) number = BigInteger.valueOf(number.shortValue());
-    boolean repr_decimal = constant_repr.contains (".") || constant_repr.contains("E") || constant_repr.contains("e");
+    boolean repr_decimal = constant_repr.contains (".") || 
+        ((constant_repr.contains("E") || constant_repr.contains("e")) &&
+            !(constant_repr.startsWith("0x") || constant_repr.startsWith("0X")));
     if (repr_decimal) {
       BigDecimal constant = SpecialConstantBitPatternTransforms.asBigDecimal(constant_repr);
       return constant.equals (number);

src/main/java/gov/nasa/pds/tools/validate/content/table/InventoryTableValidator.java

@@ -2,7 +2,7 @@
 
 import java.net.URL;
 import java.util.ArrayList;
-import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import javax.xml.xpath.XPathExpressionException;
@@ -50,20 +50,20 @@ public static void uniqueCollectionRefs (ProblemListener listener, URL aggregate
     }
   }
   private static void checkAllUnique(List<String> ids, ProblemListener listener, URL aggregate) {
-      HashSet<String> uniqueIDs = new HashSet<String>(ids);
-      if (ids.size() != uniqueIDs.size()) {
-        int dups = ids.size() - uniqueIDs.size();
-        for (String uid : uniqueIDs) {
-          int copies = Collections.frequency (ids, uid);
-          if (1 < copies) {
-            dups = dups - copies + 1;
-            listener.addProblem(new ValidationProblem(
-                new ProblemDefinition(ExceptionType.ERROR, ProblemType.INVENTORY_DUPLICATE_LIDVID,
-                    "Inventory contains " + Integer.toString(copies) + "' instances of LIDVID " + uid),
-                aggregate));
-          }
-          if (dups < 1) break;  // shortcut the for loop
-        }
+    HashSet<String> dups = new HashSet<String>();
+    HashMap<String,Integer> counts = new HashMap<String,Integer>();
+    for (String id : ids) {
+      Integer count = counts.getOrDefault(id, Integer.valueOf(0));
+      counts.put(id, ++count);
+      if (count > 1) {
+        dups.add(id);
       }
+    }
+    for (String id : dups) {
+      listener.addProblem(new ValidationProblem(
+          new ProblemDefinition(ExceptionType.ERROR, ProblemType.INVENTORY_DUPLICATE_LIDVID,
+              "Inventory contains " + Integer.toString(counts.get(id)) + " instances of LIDVID " + id),
+          aggregate));
+    }
   }
 }

src/main/java/gov/nasa/pds/tools/validate/rule/pds4/FileReferenceValidationRule.java

@@ -114,7 +114,7 @@ public void validateFileReferences() {
     DOMSource source = new DOMSource(label);
     source.setSystemId(uri.toString());
     try {
-      TreeInfo xml = LabelParser.parse(source);
+      TreeInfo xml = LabelParser.parse(source, false);
       LOG.debug("FileReferenceValidationRule:validateFileReferences:uri {}", uri);
       validate(xml.getRootNode());
     } catch (TransformerException te) {
@@ -282,15 +282,6 @@ private boolean validate(NodeInfo xml) {
                 name = child.getStringValue();
                 this.fileMapping.put(name, "");
                 LOG.debug("FileReferenceValidationRule:validate:name {}", name);
-                if (!CrossLabelFileAreaReferenceChecker.add (name, target)) {
-                  this.getListener().addProblem(
-                      new ValidationProblem(
-                          new ProblemDefinition(ExceptionType.ERROR, ProblemType.DUPLICATED_FILE_AREA_REFERENCE,
-                              "This file area references " + name + " that is already used by label "
-                              + CrossLabelFileAreaReferenceChecker.getOtherId(name, target)
-                              + " in file " + CrossLabelFileAreaReferenceChecker.getOtherFilename(name, target)),
-                          target, fileObject.getLineNumber(), -1));
-                };
               } else if ("md5_checksum".equals(child.getLocalPart())) {
                 checksum = child.getStringValue();
                 LOG.debug("FileReferenceValidationRule:validate:checksum {}", checksum);
@@ -330,6 +321,19 @@ private boolean validate(NodeInfo xml) {
                 LOG.debug("FileReferenceValidationRule:validate:filesize {}", filesize);
               }
             } // for (TinyNodeImpl child : children)
+
+            if (!name.isBlank()) {
+              String fullName = directory.isBlank() ? name : String.join(File.pathSeparator, directory, name);
+              if (!CrossLabelFileAreaReferenceChecker.add (fullName, target)) {
+                this.getListener().addProblem(
+                    new ValidationProblem(
+                        new ProblemDefinition(ExceptionType.ERROR, ProblemType.DUPLICATED_FILE_AREA_REFERENCE,
+                            "This file area references " + fullName + " that is already used by label "
+                                + CrossLabelFileAreaReferenceChecker.getOtherId(fullName, target)
+                                + " in file " + CrossLabelFileAreaReferenceChecker.getOtherFilename(fullName, target)),
+                        target, fileObject.getLineNumber(), -1));
+              }
+            }
 
             this.checkExtension(name, encodingStandardId);
 

src/main/java/gov/nasa/pds/validate/ri/UserInput.java

@@ -42,7 +42,7 @@ private boolean isLabel (String cliArg) {
       if (file.exists()) {
         try {
           SAXSource saxSource = new SAXSource(new InputSource(new FileReader(file)));
-          TreeInfo docInfo = LabelParser.parse(saxSource); // Parses a label.
+          TreeInfo docInfo = LabelParser.parse(saxSource, true); // Parses a label.
           List<TinyNodeImpl> xmlModels = new ArrayList<>();
           XMLExtractor extractor = new XMLExtractor(docInfo.getRootNode());
           xmlModels = extractor.getNodesFromDoc("//logical_identifier");

src/test/resources/features/developer.feature

@@ -9,6 +9,15 @@ Scenario Outline: Execute validate command for tests below.
   Examples:
  | testName                                             | testDir     | messageCount | messageText         | problemEnum     | resourceDir          | reportDir     | commandArgs                                                                | refOutputValue | 
 
+# Validate#873
+|"NASA-PDS/validate#873 Success files same name different paths" | "github873" | 0 | "0 errors expected" | "totalErrors" | "src/test/resources"   | "target/test" | "-r {reportDir}/report_github873.json -s json --skip-context-validation -R pds4.bundle -t {resourceDir}/github873" | "report_github873.json" |
+
+# Validate#849
+|"NASA-PDS/validate#849 Fails inventory with duplicate entries" | "github849" | 2 | "2 errors expected" | "INVENTORY_DUPLICATE_LIDVID" | "src/test/resources"   | "target/test" | "-r {reportDir}/report_github849.json -s json --skip-context-validation  -t {resourceDir}/github849/collection_uvs_data_raw.xml" | "report_github849.json" |
+
+# Validate#831
+|"NASA-PDS/validate#831 Success high instrument saturation" | "github831" | 0 | "0 errors expected" | "totalErrors" | "src/test/resources"   | "target/test" | "-r {reportDir}/report_github831.json -s json --skip-context-validation  -t {resourceDir}/github831/kplo.xml" | "report_github831.json" |
+
 # Validate#824
 |"NASA-PDS/validate#824 Success PDF vs PDF/A" | "github824" | 0 | "0 errors expected" | "totalErrors" | "src/test/resources"   | "target/test" | "-r {reportDir}/report_github824.json -s json --skip-context-validation  -t {resourceDir}/github824/1203_12.xml" | "report_github824.json" |
 

src/test/resources/github831/kplo.xml

@@ -0,0 +1,154 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1L00.sch" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<?xml-model href="https://pds.nasa.gov/pds4/disp/v1/PDS4_DISP_1L00_1510.sch" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<?xml-model href="https://pds.nasa.gov/pds4/img/v1/PDS4_IMG_1L00_1890.sch" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<?xml-model href="https://pds.nasa.gov/pds4/geom/v1/PDS4_GEOM_1L00_1970.sch" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+
+<Product_Observational
+  xmlns="http://pds.nasa.gov/pds4/pds/v1"
+  xmlns:disp="http://pds.nasa.gov/pds4/disp/v1"
+  xmlns:img="http://pds.nasa.gov/pds4/img/v1"
+  xmlns:geom="http://pds.nasa.gov/pds4/geom/v1"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+
+  xsi:schemaLocation="http://pds.nasa.gov/pds4/pds/v1 https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1L00.xsd
+  http://pds.nasa.gov/pds4/disp/v1 https://pds.nasa.gov/pds4/disp/v1/PDS4_DISP_1L00_1510.xsd
+  http://pds.nasa.gov/pds4/img/v1 https://pds.nasa.gov/pds4/img/v1/PDS4_IMG_1L00_1890.xsd
+  http://pds.nasa.gov/pds4/geom/v1 https://pds.nasa.gov/pds4/geom/v1/PDS4_GEOM_1L00_1970.xsd">
+
+  <Identification_Area>
+    <logical_identifier>urn:nasa:pds:kplo-shadowcam:observation:e004246319sc</logical_identifier>
+    <version_id>1.0</version_id>
+    <title>Korea Pathfinder Lunar Orbiter ShadowCam Calibrated Product</title>
+    <information_model_version>1.21.0.0</information_model_version>
+    <product_class>Product_Observational</product_class>
+    <Modification_History>
+      <Modification_Detail>
+        <modification_date>2024-01-27T22:53:08.112700Z</modification_date>
+        <version_id>1.0</version_id>
+        <description>
+          Initial Release
+        </description>
+      </Modification_Detail>
+    </Modification_History>
+  </Identification_Area>
+  <Observation_Area>
+    <Time_Coordinates>
+      <start_date_time>2022-09-23T02:39:59.931890Z</start_date_time>
+      <stop_date_time>2022-09-23T02:40:42.952997Z</stop_date_time>
+    </Time_Coordinates>
+    <Primary_Result_Summary>
+      <purpose>Science</purpose>
+      <processing_level>Calibrated</processing_level>
+      <description>ShadowCam is a Time Delay Integration(TDI) pushbroom imager 
+      </description>
+    </Primary_Result_Summary>
+    <Investigation_Area>
+      <name>KOREA PATHFINDER LUNAR ORBITER</name>
+      <type>Mission</type>
+      <Internal_Reference>
+        <lid_reference>urn:kari:kpds:context:investigation:mission.kplo</lid_reference>
+        <reference_type>data_to_investigation</reference_type>
+        <comment>This is the PDS4 logical identifier for KPLO</comment>
+      </Internal_Reference>
+    </Investigation_Area>
+    <Observing_System>
+      <Observing_System_Component>
+        <name>Korea Pathfinder Lunar Orbiter (Danuri)</name>
+        <type>Host</type>
+        <Internal_Reference>
+          <lid_reference>urn:kari:kpds:context:instrument_host:spacecraft.kplo</lid_reference>
+          <reference_type>is_instrument_host</reference_type>
+        </Internal_Reference>
+      </Observing_System_Component>
+      <Observing_System_Component>
+        <name>SHADOWCAM</name>
+        <type>Instrument</type>
+        <Internal_Reference>
+          <lid_reference>urn:kari:kpds:context:instrument:kplo.shc</lid_reference>
+          <reference_type>is_instrument</reference_type>
+        </Internal_Reference>
+      </Observing_System_Component>
+    </Observing_System>
+    <Target_Identification>
+      <name>Earth</name>
+      <type>Planet</type>
+      <Internal_Reference>
+        <lid_reference>urn:nasa:pds:context:target:planet.earth</lid_reference>
+        <reference_type>data_to_target</reference_type>
+      </Internal_Reference>
+    </Target_Identification>
+    <Target_Identification>
+      <name>Moon</name>
+      <type>Satellite</type>
+      <Internal_Reference>
+        <lid_reference>urn:nasa:pds:context:target:satellite.earth.moon</lid_reference>
+        <reference_type>data_to_target</reference_type>
+      </Internal_Reference>
+    </Target_Identification>
+    <Mission_Area>
+    </Mission_Area>
+    <Discipline_Area>
+      <img:Imaging>
+        <Local_Internal_Reference>
+          <local_identifier_reference>DATA_FILE</local_identifier_reference>
+          <local_reference_type>imaging_parameters_to_image_object</local_reference_type>
+        </Local_Internal_Reference>
+        <img:Optical_Filter>
+          <img:bandwidth unit="nm">300</img:bandwidth>
+          <img:center_filter_wavelength unit="nm">600</img:center_filter_wavelength>
+        </img:Optical_Filter>
+      </img:Imaging>
+      <disp:Display_Settings>
+        <Local_Internal_Reference>
+          <local_identifier_reference>Array_2D_Image</local_identifier_reference>
+          <local_reference_type>display_settings_to_array</local_reference_type>
+        </Local_Internal_Reference>
+        <disp:Display_Direction>
+          <disp:horizontal_display_axis>Sample</disp:horizontal_display_axis>
+          <disp:horizontal_display_direction>Left to Right</disp:horizontal_display_direction>
+          <disp:vertical_display_axis>Line</disp:vertical_display_axis>
+          <disp:vertical_display_direction>Top to Bottom</disp:vertical_display_direction>
+        </disp:Display_Direction>
+      </disp:Display_Settings>
+    </Discipline_Area>
+  </Observation_Area>
+  <File_Area_Observational>
+    <File>
+      <file_name>kplo.cub</file_name>
+      <local_identifier>DATA_FILE</local_identifier>
+      <creation_date_time>2024-02-02T23:19:08.197538Z</creation_date_time>
+      <file_size unit="byte">513</file_size>
+      <!--md5_checksum>80393407b982ed98f6f1ce99f5af7638</md5_checksum-->
+      <comment>Korea Pathfinder Lunar Orbiter Calibrated Product.</comment>
+    </File>
+    <Array_2D_Image>
+      <local_identifier>Array_2D_Image</local_identifier>
+      <offset unit="byte">0</offset>
+      <axes>2</axes>
+      <axis_index_order>Last Index Fastest</axis_index_order>
+      <Element_Array>
+        <data_type>IEEE754LSBSingle</data_type>
+        <unit>Radiance</unit>
+      </Element_Array>
+      <Axis_Array>
+        <axis_name>Line</axis_name>
+        <elements>16</elements>
+        <sequence_number>1</sequence_number>
+      </Axis_Array>
+      <Axis_Array>
+        <axis_name>Sample</axis_name>
+        <elements>8</elements>
+        <sequence_number>2</sequence_number>
+      </Axis_Array>
+      <Special_Constants>
+        <missing_constant>0xFF7FFFFB</missing_constant>
+        <high_instrument_saturation>0xFF7FFFFE</high_instrument_saturation>
+        <high_representation_saturation>0xFF7FFFFF</high_representation_saturation>
+        <valid_minimum>0xFF7FFFFA</valid_minimum>
+        <low_instrument_saturation>0xFF7FFFFD</low_instrument_saturation>
+        <low_representation_saturation>0xFF7FFFFC</low_representation_saturation>
+      </Special_Constants>
+    </Array_2D_Image>
+  </File_Area_Observational>
+</Product_Observational>