StringMap bug when using an emoji string parsed from JSON in C++ backend

[singpolyma]

import haxe.Json;
import haxe.ds.StringMap;

class ReproEmojiStringMap {
	static function dump(label:String, s:String) {
		trace(label, s, s.length, [for (i in 0...s.length) s.charCodeAt(i)]);
	}

	static function main() {
		// Get the same string from json and a literal
		var fromJson:String = cast Json.parse("\"\\ud83d\\udc4d\"");
		var literal = "👍";

		// confirm they are the same string
		dump("fromJson", fromJson);
		dump("literal", literal);
		trace("equal", fromJson == literal);

		// Set a StringMap key using the one from JSON
		var m = new StringMap<Int>();
		m.set(fromJson, 1);

		// Observe that the identical string literal does not work to look up this key
		trace("exists(fromJson)", m.exists(fromJson));
		trace("exists(literal)", m.exists(literal));
		trace("get(fromJson)", m.get(fromJson));
		trace("get(literal)", m.get(literal));
	}
}

[singpolyma]

It appears there are actually at least three possible different hash values generated for this same one string:

  import haxe.Json;
  import haxe.io.Bytes;

  class ReproEmojiHashes {
    static function normalize(s:String):String {
      return Bytes.ofString(s).toString();
    }

    static function dump(label:String, s:String) {
      var hash:Int = untyped __cpp__("{0}.hash()", s);
      var utf16:Bool = untyped __cpp__("{0}.isUTF16Encoded()", s);
      trace(label, hash, utf16);
    }

    static function main() {
      var literal = "👍";
      var fromJson:String = cast Json.parse("\"\\ud83d\\udc4d\"");
      var normalizedLiteral = normalize(literal);
      var normalizedJson = normalize(fromJson);

      dump("literal", literal);
      dump("fromJson", fromJson);
      dump("normalizedLiteral", normalizedLiteral);
      dump("normalizedJson", normalizedJson);
    }
  }

[singpolyma]

There's also no need to get fancy with surrogates to reproduce this, var fromJson:String = cast Json.parse("\"👍\""); does as well

[tobil4sk]

Haxe is actually generating the correct hash for the literal. The hxcpp runtime has two bugs that:

1. generates the hash incorrectly for non-BMP utf16 strings (breaking fromJson)
2. reads the hash incorrectly for any utf16 strings converted from bytes (breaking the normalized hashes)

@Simn Since there is no bug on the Haxe generator side, could this be transfered to the hxcpp repo? I have patches I will send soon.