fix: preserve original metadata offsets for CRLF input

spec/endIndex_spec.js

@@ -19,8 +19,8 @@ function collectMeta(node, out = []) {
 }
 
 /** Parse xml with metadata capture on and return a Map of rawTagName -> raw text span. */
-function parseSpans(xml) {
-  const parser = new XMLParser({ preserveOrder: true, ignoreAttributes: false, captureMetaData: true });
+function parseSpans(xml, options = {}) {
+  const parser = new XMLParser(Object.assign({ preserveOrder: true, ignoreAttributes: false, captureMetaData: true }, options));
   const result = parser.parse(xml);
   return new Map(collectMeta(result).map(([tag, m]) => [tag, xml.slice(m.startIndex, m.endIndex)]));
 }
@@ -94,4 +94,56 @@ describe("XMLParser captureMetaData endIndex", function () {
         expect(byTag.get("b")).toBe(`<b>x</b>`);
         expect(byTag.get("a")).toBe(xml);
     });
+
+    for (const newline of ["\n", "\r", "\r\n", "\r\n\n\r"]) {
+        it("uses original input offsets with " + JSON.stringify(newline) + " line endings", function () {
+            const declaration = `<?xml${newline}version="1.0"?>`;
+            const child = `<child${newline}id="1">first${newline}second</child>`;
+            const empty = `<empty${newline}/>`;
+            const pi = `<?pi${newline}value="x"?>`;
+            const root = `<root><!--first${newline}second--><![CDATA[first${newline}second]]>${child}${empty}${pi}</root>`;
+            const spans = parseSpans(declaration + newline + root);
+
+            expect(spans.get("?xml")).toBe(declaration);
+            expect(spans.get("root")).toBe(root);
+            expect(spans.get("child")).toBe(child);
+            expect(spans.get("empty")).toBe(empty);
+            expect(spans.get("?pi")).toBe(pi);
+        });
+    }
+
+    it("uses original offsets for stop nodes, unpaired tags and retained siblings", function () {
+        const stop = "<stop>first\r\n<b>raw</b>\r\nlast</stop>";
+        const br = '<br\r\nclass="x">';
+        const xml = `<root>\r\n${stop}\r\n${br}\r\n<skip/>\r\n</root>`;
+        const spans = parseSpans(xml, {
+            stopNodes: ["root.stop"],
+            unpairedTags: ["br"],
+            updateTag: (tagName) => tagName === "skip" ? false : tagName,
+        });
+
+        expect(spans.get("root")).toBe(xml);
+        expect(spans.get("stop")).toBe(stop);
+        expect(spans.get("br")).toBe(br);
+        expect(spans.has("skip")).toBeFalse();
+    });
+
+    for (const inputType of ["string", "Buffer", "Uint8Array"]) {
+        it("keeps original offsets and normalizes values for " + inputType + " input", function () {
+            const xml = '<root>\r\n<child attr="a\r\nb">first\r\nsecond</child>\r\n</root>';
+            const input = inputType === "string" ? xml
+                : inputType === "Buffer" ? Buffer.from(xml) : new TextEncoder().encode(xml);
+            const result = new XMLParser({
+                ignoreAttributes: false,
+                captureMetaData: true,
+                alwaysCreateTextNode: true,
+            }).parse(input);
+            const metadata = result.root.child[XML_METADATA];
+
+            expect(xml.slice(metadata.startIndex, metadata.endIndex))
+                .toBe('<child attr="a\r\nb">first\r\nsecond</child>');
+            expect(result.root.child["@_attr"]).toBe("a\nb");
+            expect(result.root.child["#text"]).toBe("first\nsecond");
+        });
+    }
 });

src/xmlparser/OrderedObjParser.js

@@ -283,7 +283,15 @@ function buildAttributesMap(attrStr, jPath, tagName, force = false) {
   }
 }
 const parseXml = function (xmlData) {
-  xmlData = xmlData.replace(/\r\n?/g, "\n"); //TODO: remove this line
+  const options = this.options;
+  const crlfPositions = options.captureMetaData ? [] : undefined;
+  xmlData = xmlData.replace(/\r\n?/g, options.captureMetaData ? (newline, index) => {
+    if (newline.length === 2) {
+      // Store the position after earlier CRLF pairs have been normalized.
+      crlfPositions.push(index - crlfPositions.length);
+    }
+    return "\n";
+  } : "\n");
   const xmlObj = new xmlNode('!xml');
   let currentNode = xmlObj;
   let textData = "";
@@ -296,7 +304,6 @@ const parseXml = function (xmlData) {
   this.entityExpansionCount = 0;
   this.currentExpandedLength = 0;
   this.doctypefound = false;
-  const options = this.options;
   const docTypeReader = new DocTypeReader(options.processEntities);
   const xmlLen = xmlData.length;
   for (let i = 0; i < xmlLen; i++) {//for each char in XML data
@@ -340,7 +347,7 @@ const parseXml = function (xmlData) {
         currentNode = this.tagsNodeStack.pop() || xmlObj;//avoid recursion, set the parent tag scope
 
         if (options.captureMetaData && currentNode) {
-          currentNode.addEndIndex(closeIndex + 1);
+          currentNode.addEndIndex(getOriginalIndex(closeIndex + 1, crlfPositions));
         }
         textData = "";
         i = closeIndex;
@@ -366,11 +373,12 @@ const parseXml = function (xmlData) {
           if (tagData.tagName !== tagData.tagExp && tagData.attrExpPresent && options.ignoreAttributes !== true) {
             childNode[":@"] = attsMap
           }
-          this.addChild(currentNode, childNode, this.readonlyMatcher, i);
+          const startIndex = options.captureMetaData ? getOriginalIndex(i, crlfPositions) : i;
+          this.addChild(currentNode, childNode, this.readonlyMatcher, startIndex);
 
           if (options.captureMetaData) {
             // closeIndex points at '?' of the closing '?>'
-            currentNode.addEndIndex(tagData.closeIndex + 2);
+            currentNode.addEndIndex(getOriginalIndex(tagData.closeIndex + 2, crlfPositions));
           }
         }
 
@@ -502,7 +510,7 @@ const parseXml = function (xmlData) {
           this.isCurrentNodeStopNode = this.isItStopNode();
         }
 
-        const startIndex = i;
+        const startIndex = options.captureMetaData ? getOriginalIndex(i, crlfPositions) : i;
         if (this.isCurrentNodeStopNode) {
           let tagContent = "";
 
@@ -538,7 +546,7 @@ const parseXml = function (xmlData) {
           this.addChild(currentNode, childNode, this.readonlyMatcher, startIndex);
 
           if (options.captureMetaData) {
-            currentNode.addEndIndex(i + 1);
+            currentNode.addEndIndex(getOriginalIndex(i + 1, crlfPositions));
           }
         } else {
           //selfClosing tag
@@ -552,7 +560,7 @@ const parseXml = function (xmlData) {
             this.addChild(currentNode, childNode, this.readonlyMatcher, startIndex);
 
             if (options.captureMetaData) {
-              currentNode.addEndIndex(closeIndex + 1);
+              currentNode.addEndIndex(getOriginalIndex(closeIndex + 1, crlfPositions));
             }
             this.matcher.pop(); // Pop self-closing tag
             this.isCurrentNodeStopNode = false; // Reset flag
@@ -565,7 +573,7 @@ const parseXml = function (xmlData) {
             this.addChild(currentNode, childNode, this.readonlyMatcher, startIndex);
 
             if (options.captureMetaData) {
-              currentNode.addEndIndex(result.closeIndex + 1);
+              currentNode.addEndIndex(getOriginalIndex(result.closeIndex + 1, crlfPositions));
             }
             this.matcher.pop(); // Pop unpaired tag
             this.isCurrentNodeStopNode = false; // Reset flag
@@ -598,6 +606,17 @@ const parseXml = function (xmlData) {
   return xmlObj.child;
 }
 
+function getOriginalIndex(index, crlfPositions) {
+  let low = 0;
+  let high = crlfPositions.length;
+  while (low < high) {
+    const middle = Math.floor((low + high) / 2);
+    if (crlfPositions[middle] < index) low = middle + 1;
+    else high = middle;
+  }
+  return index + low;
+}
+
 function addChild(currentNode, childNode, matcher, startIndex) {
   // unset startIndex if not requested
   if (!this.options.captureMetaData) startIndex = undefined;
@@ -868,4 +887,4 @@ function sanitizeName(name, options) {
     return options.onDangerousProperty(name);
   }
   return name;
-}
+}