[pull] master from sqlmapproject:master

lib/core/common.py

@@ -4044,7 +4044,7 @@ def decodeIntToUnicode(value):
                     # Reference: https://docs.microsoft.com/en-us/sql/relational-databases/collations/collation-and-unicode-support?view=sql-server-2017 and https://stackoverflow.com/a/14488478
                     # supplementary codepoints (>0xFFFF, _SC collations) aren't 2-byte UTF-16; decode direct
                     retVal = _unichr(value) if value > 0xFFFF else getUnicode(raw, "UTF-16-BE")
-                elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE, DBMS.SQLITE, DBMS.DB2, DBMS.PRESTO, DBMS.H2, DBMS.HSQLDB, DBMS.DERBY, DBMS.MONETDB, DBMS.VERTICA):     # Note: cases with Unicode code points (e.g. http://www.postgresqltutorial.com/postgresql-ascii/)
+                elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE, DBMS.SQLITE, DBMS.DB2, DBMS.PRESTO, DBMS.H2, DBMS.HSQLDB, DBMS.DERBY, DBMS.MONETDB, DBMS.VERTICA, DBMS.SPANNER):     # Note: cases with Unicode code points (e.g. http://www.postgresqltutorial.com/postgresql-ascii/); Spanner via TO_CODE_POINTS
                     retVal = _unichr(value)
                 else:
                     retVal = getUnicode(raw, conf.encoding)

lib/core/convert.py

@@ -637,7 +637,14 @@ def stdoutEncode(value):
 
         kb.codePage = kb.codePage or ""
 
-    encoding = kb.get("codePage") or getattr(sys.stdout, "encoding", None) or UNICODE_ENCODING
+    # Python 3.6+ writes to the Windows console via WriteConsoleW (PEP 528), so sys.stdout.encoding
+    # (typically 'utf-8') renders Unicode that the legacy OEM/ANSI code page from 'chcp' would replace
+    # with '?'; prefer it there. The chcp-derived code page stays as the fallback (older interpreters,
+    # PYTHONLEGACYWINDOWSSTDIO, or a console encoding Python could not determine) and for Python 2.
+    if six.PY3:
+        encoding = getattr(sys.stdout, "encoding", None) or kb.get("codePage") or UNICODE_ENCODING
+    else:
+        encoding = kb.get("codePage") or getattr(sys.stdout, "encoding", None) or UNICODE_ENCODING
 
     if six.PY3:
         if isinstance(value, (bytes, bytearray)):

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.250"
+VERSION = "1.10.7.253"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/request/inject.py

@@ -594,6 +594,34 @@ def _(item):
     applyFunctionRecursively(value, _)
     return retVal[0]
 
+def _looksLikeMisdecodedUtf8(value):
+    """
+    True if a retrieved value looks like UTF-8 data shown through a single-byte (ISO-8859-1) charset -
+    the '<page charset too WIDE>' mismatch that leaves NO undecodable bytes (latin-1 accepts every byte),
+    so _pageCharsetCorrupted() cannot see it, yet the data is silently mojibake (a UTF-8 accent shows as
+    two spurious latin-1 chars).
+
+    Precise by construction: real mojibake re-encodes to latin-1 and decodes back to a DIFFERENT valid
+    UTF-8 string, whereas correctly-decoded UTF-8 ('cafe' with U+00E9) and genuine latin-1 text ('resume')
+    do not (a lone accent is a UTF-8 lead byte with no continuation -> decode fails). So it fires only on
+    the corruption, not on clean data. Complements _pageCharsetCorrupted() (the too-NARROW direction).
+    """
+
+    retVal = [False]
+
+    def _(item):
+        if not retVal[0] and isinstance(item, six.string_types) and any(0x80 <= ord(_) <= 0xFF for _ in item):
+            try:
+                decoded = item.encode("latin-1").decode("utf-8")
+            except (UnicodeEncodeError, UnicodeDecodeError):
+                return item
+            if decoded != item and any(ord(_) > 0x7F for _ in decoded):
+                retVal[0] = True
+        return item
+
+    applyFunctionRecursively(value, _)
+    return retVal[0]
+
 @lockedmethod
 @stackedmethod
 def getValue(expression, blind=True, union=True, error=True, time=True, fromUser=False, expected=None, batch=False, unpack=True, resumeValue=True, charsetType=None, firstChar=None, lastChar=None, dump=False, suppressOutput=None, expectingNone=False, safeCharEncode=True):
@@ -699,7 +727,7 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
                 if (found and not conf.hexConvert and not conf.binaryFields and expected not in (EXPECTED.BOOL, EXPECTED.INT)
                         and getTechnique() in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)
                         and Backend.getIdentifiedDbms() and hasattr(queries[Backend.getIdentifiedDbms()], "hex")
-                        and _pageCharsetCorrupted(value)):
+                        and (_pageCharsetCorrupted(value) or _looksLikeMisdecodedUtf8(value))):
                     warnMsg = "retrieved data appears corrupted because of a charset mismatch between the "
                     warnMsg += "DBMS and the web page. Re-fetching using hexadecimal encoding"
                     singleTimeWarnMessage(warnMsg)
@@ -710,7 +738,10 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
                     finally:
                         conf.hexConvert = False
 
-                    if _value is not None:
+                    # only adopt the hex re-fetch if it is actually cleaner: a rare mis-trigger (or a
+                    # column whose real charset the hex decode still cannot resolve) must never replace
+                    # the original with equal-or-worse data
+                    if _value is not None and not _pageCharsetCorrupted(_value) and not _looksLikeMisdecodedUtf8(_value):
                         value = _value
 
                 if found and conf.dnsDomain:

tests/test_charset.py

@@ -22,6 +22,7 @@
 bootstrap()
 
 from lib.request.basic import checkCharEncoding
+from lib.request.inject import _pageCharsetCorrupted, _looksLikeMisdecodedUtf8
 from lib.core.common import extractRegexResult, paramToDict
 from lib.core.enums import PLACE
 from lib.core.settings import META_CHARSET_REGEX, HTML_TITLE_REGEX, META_REFRESH_REGEX
@@ -58,6 +59,38 @@ def test_no_match_is_none(self):
         self.assertIsNone(extractRegexResult(HTML_TITLE_REGEX, "<body>no title here</body>"))
 
 
+class TestCharsetMismatchDetection(unittest.TestCase):
+    """The auto-hex recovery triggers on two complementary mismatch directions:
+    _pageCharsetCorrupted (page charset too NARROW -> undecodable bytes) and
+    _looksLikeMisdecodedUtf8 (page charset too WIDE -> UTF-8 shown as latin-1, no
+    undecodable byte). Both must fire on the corruption and NOT on clean data."""
+
+    # correctly-decoded UTF-8 and genuine latin-1 text: neither detector may fire (no wasted re-fetch).
+    # \u escapes keep this source pure-ASCII (py2): cafe / naive / Zurich / CJK / Cyrillic / resume / garcon
+    CLEAN = [u"admin", u"caf\u00e9", u"na\u00efve", u"Z\u00fcrich", u"\u65e5\u672c\u8a9e",
+             u"\u0417\u0434\u0440\u0430\u0432\u0435\u0439", u"r\u00e9sum\u00e9", u"gar\u00e7on"]
+
+    def test_clean_values_trigger_neither(self):
+        for v in self.CLEAN:
+            self.assertFalse(_pageCharsetCorrupted(v), msg="narrow FP: %r" % v)
+            self.assertFalse(_looksLikeMisdecodedUtf8(v), msg="wide FP: %r" % v)
+
+    def test_utf8_shown_as_latin1_is_caught(self):
+        # gap #1: UTF-8 column bytes decoded as latin-1 -> valid mojibake, no undecodable byte
+        for word in (u"caf\u00e9", u"\u65e5\u672c\u8a9e", u"\u0417\u0434\u0440\u0430\u0432\u0435\u0439", u"\u20ac"):
+            mojibake = word.encode("utf-8").decode("latin-1")
+            self.assertFalse(_pageCharsetCorrupted(mojibake), msg="narrow should miss: %r" % mojibake)
+            self.assertTrue(_looksLikeMisdecodedUtf8(mojibake), msg="wide should catch: %r" % mojibake)
+
+    def test_undecodable_bytes_still_caught_by_narrow(self):
+        # the other direction (page charset too narrow) leaves reversible \xNN escapes
+        self.assertTrue(_pageCharsetCorrupted(u"foo\\xe9bar"))
+
+    def test_list_and_nonstring_inputs(self):
+        self.assertTrue(_looksLikeMisdecodedUtf8([u"ok", u"caf\u00c3\u00a9"]))   # 'cafe' mojibake
+        self.assertFalse(_looksLikeMisdecodedUtf8([u"ok", 123, None]))
+
+
 class TestParamToDict(unittest.TestCase):
     # NOTE: GET parsing is covered in test_urls.py; here we only cover the COOKIE place,
     # which uses a different (semicolon) delimiter and is a distinct code path.