diff --git a/lib/core/common.py b/lib/core/common.py index ab7cbe4e9c1..9580d2bac0f 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -4044,7 +4044,7 @@ def decodeIntToUnicode(value): # Reference: https://docs.microsoft.com/en-us/sql/relational-databases/collations/collation-and-unicode-support?view=sql-server-2017 and https://stackoverflow.com/a/14488478 # supplementary codepoints (>0xFFFF, _SC collations) aren't 2-byte UTF-16; decode direct retVal = _unichr(value) if value > 0xFFFF else getUnicode(raw, "UTF-16-BE") - elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE, DBMS.SQLITE, DBMS.DB2, DBMS.PRESTO, DBMS.H2, DBMS.HSQLDB, DBMS.DERBY, DBMS.MONETDB, DBMS.VERTICA): # Note: cases with Unicode code points (e.g. http://www.postgresqltutorial.com/postgresql-ascii/) + elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE, DBMS.SQLITE, DBMS.DB2, DBMS.PRESTO, DBMS.H2, DBMS.HSQLDB, DBMS.DERBY, DBMS.MONETDB, DBMS.VERTICA, DBMS.SPANNER): # Note: cases with Unicode code points (e.g. http://www.postgresqltutorial.com/postgresql-ascii/); Spanner via TO_CODE_POINTS retVal = _unichr(value) else: retVal = getUnicode(raw, conf.encoding) diff --git a/lib/core/convert.py b/lib/core/convert.py index 39d7028dc9f..f3927e60d64 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -637,7 +637,14 @@ def stdoutEncode(value): kb.codePage = kb.codePage or "" - encoding = kb.get("codePage") or getattr(sys.stdout, "encoding", None) or UNICODE_ENCODING + # Python 3.6+ writes to the Windows console via WriteConsoleW (PEP 528), so sys.stdout.encoding + # (typically 'utf-8') renders Unicode that the legacy OEM/ANSI code page from 'chcp' would replace + # with '?'; prefer it there. The chcp-derived code page stays as the fallback (older interpreters, + # PYTHONLEGACYWINDOWSSTDIO, or a console encoding Python could not determine) and for Python 2. + if six.PY3: + encoding = getattr(sys.stdout, "encoding", None) or kb.get("codePage") or UNICODE_ENCODING + else: + encoding = kb.get("codePage") or getattr(sys.stdout, "encoding", None) or UNICODE_ENCODING if six.PY3: if isinstance(value, (bytes, bytearray)): diff --git a/lib/core/settings.py b/lib/core/settings.py index c6797c85727..93273ac4c37 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.250" +VERSION = "1.10.7.253" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/request/inject.py b/lib/request/inject.py index 0af59544fcc..0c46ba173e9 100644 --- a/lib/request/inject.py +++ b/lib/request/inject.py @@ -594,6 +594,34 @@ def _(item): applyFunctionRecursively(value, _) return retVal[0] +def _looksLikeMisdecodedUtf8(value): + """ + True if a retrieved value looks like UTF-8 data shown through a single-byte (ISO-8859-1) charset - + the '' mismatch that leaves NO undecodable bytes (latin-1 accepts every byte), + so _pageCharsetCorrupted() cannot see it, yet the data is silently mojibake (a UTF-8 accent shows as + two spurious latin-1 chars). + + Precise by construction: real mojibake re-encodes to latin-1 and decodes back to a DIFFERENT valid + UTF-8 string, whereas correctly-decoded UTF-8 ('cafe' with U+00E9) and genuine latin-1 text ('resume') + do not (a lone accent is a UTF-8 lead byte with no continuation -> decode fails). So it fires only on + the corruption, not on clean data. Complements _pageCharsetCorrupted() (the too-NARROW direction). + """ + + retVal = [False] + + def _(item): + if not retVal[0] and isinstance(item, six.string_types) and any(0x80 <= ord(_) <= 0xFF for _ in item): + try: + decoded = item.encode("latin-1").decode("utf-8") + except (UnicodeEncodeError, UnicodeDecodeError): + return item + if decoded != item and any(ord(_) > 0x7F for _ in decoded): + retVal[0] = True + return item + + applyFunctionRecursively(value, _) + return retVal[0] + @lockedmethod @stackedmethod def getValue(expression, blind=True, union=True, error=True, time=True, fromUser=False, expected=None, batch=False, unpack=True, resumeValue=True, charsetType=None, firstChar=None, lastChar=None, dump=False, suppressOutput=None, expectingNone=False, safeCharEncode=True): @@ -699,7 +727,7 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser if (found and not conf.hexConvert and not conf.binaryFields and expected not in (EXPECTED.BOOL, EXPECTED.INT) and getTechnique() in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY) and Backend.getIdentifiedDbms() and hasattr(queries[Backend.getIdentifiedDbms()], "hex") - and _pageCharsetCorrupted(value)): + and (_pageCharsetCorrupted(value) or _looksLikeMisdecodedUtf8(value))): warnMsg = "retrieved data appears corrupted because of a charset mismatch between the " warnMsg += "DBMS and the web page. Re-fetching using hexadecimal encoding" singleTimeWarnMessage(warnMsg) @@ -710,7 +738,10 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser finally: conf.hexConvert = False - if _value is not None: + # only adopt the hex re-fetch if it is actually cleaner: a rare mis-trigger (or a + # column whose real charset the hex decode still cannot resolve) must never replace + # the original with equal-or-worse data + if _value is not None and not _pageCharsetCorrupted(_value) and not _looksLikeMisdecodedUtf8(_value): value = _value if found and conf.dnsDomain: diff --git a/tests/test_charset.py b/tests/test_charset.py index b7930bee061..9ef15fcf890 100644 --- a/tests/test_charset.py +++ b/tests/test_charset.py @@ -22,6 +22,7 @@ bootstrap() from lib.request.basic import checkCharEncoding +from lib.request.inject import _pageCharsetCorrupted, _looksLikeMisdecodedUtf8 from lib.core.common import extractRegexResult, paramToDict from lib.core.enums import PLACE from lib.core.settings import META_CHARSET_REGEX, HTML_TITLE_REGEX, META_REFRESH_REGEX @@ -58,6 +59,38 @@ def test_no_match_is_none(self): self.assertIsNone(extractRegexResult(HTML_TITLE_REGEX, "no title here")) +class TestCharsetMismatchDetection(unittest.TestCase): + """The auto-hex recovery triggers on two complementary mismatch directions: + _pageCharsetCorrupted (page charset too NARROW -> undecodable bytes) and + _looksLikeMisdecodedUtf8 (page charset too WIDE -> UTF-8 shown as latin-1, no + undecodable byte). Both must fire on the corruption and NOT on clean data.""" + + # correctly-decoded UTF-8 and genuine latin-1 text: neither detector may fire (no wasted re-fetch). + # \u escapes keep this source pure-ASCII (py2): cafe / naive / Zurich / CJK / Cyrillic / resume / garcon + CLEAN = [u"admin", u"caf\u00e9", u"na\u00efve", u"Z\u00fcrich", u"\u65e5\u672c\u8a9e", + u"\u0417\u0434\u0440\u0430\u0432\u0435\u0439", u"r\u00e9sum\u00e9", u"gar\u00e7on"] + + def test_clean_values_trigger_neither(self): + for v in self.CLEAN: + self.assertFalse(_pageCharsetCorrupted(v), msg="narrow FP: %r" % v) + self.assertFalse(_looksLikeMisdecodedUtf8(v), msg="wide FP: %r" % v) + + def test_utf8_shown_as_latin1_is_caught(self): + # gap #1: UTF-8 column bytes decoded as latin-1 -> valid mojibake, no undecodable byte + for word in (u"caf\u00e9", u"\u65e5\u672c\u8a9e", u"\u0417\u0434\u0440\u0430\u0432\u0435\u0439", u"\u20ac"): + mojibake = word.encode("utf-8").decode("latin-1") + self.assertFalse(_pageCharsetCorrupted(mojibake), msg="narrow should miss: %r" % mojibake) + self.assertTrue(_looksLikeMisdecodedUtf8(mojibake), msg="wide should catch: %r" % mojibake) + + def test_undecodable_bytes_still_caught_by_narrow(self): + # the other direction (page charset too narrow) leaves reversible \xNN escapes + self.assertTrue(_pageCharsetCorrupted(u"foo\\xe9bar")) + + def test_list_and_nonstring_inputs(self): + self.assertTrue(_looksLikeMisdecodedUtf8([u"ok", u"caf\u00c3\u00a9"])) # 'cafe' mojibake + self.assertFalse(_looksLikeMisdecodedUtf8([u"ok", 123, None])) + + class TestParamToDict(unittest.TestCase): # NOTE: GET parsing is covered in test_urls.py; here we only cover the COOKIE place, # which uses a different (semicolon) delimiter and is a distinct code path.