[pull] master from sqlmapproject:master

lib/core/common.py

@@ -1967,19 +1967,21 @@ def getLimitRange(count, plusOne=False):
 
     if kb.dumpTable:
         if conf.limitStart and conf.limitStop and conf.limitStart > conf.limitStop:
-            limitStop = conf.limitStart
+            limitStop = min(conf.limitStart, count)  # a '--start' beyond the table must not request out-of-range offsets (phantom rows)
             limitStart = conf.limitStop
             reverse = True
         else:
             if isinstance(conf.limitStop, int) and conf.limitStop > 0 and conf.limitStop < limitStop:
                 limitStop = conf.limitStop
 
-            if isinstance(conf.limitStart, int) and conf.limitStart > 0 and conf.limitStart <= limitStop:
+            # NOTE: no '<= limitStop' gate - a '--start' past the row count must yield an EMPTY range
+            # (correctly skipping past every row), not silently fall back to dumping the whole table
+            if isinstance(conf.limitStart, int) and conf.limitStart > 0:
                 limitStart = conf.limitStart
 
     retVal = xrange(limitStart, limitStop + 1) if plusOne else xrange(limitStart - 1, limitStop)
 
-    if reverse:
+    if reverse and len(retVal):  # len() guard: a clamped out-of-range '--start' can leave the range empty
         retVal = xrange(retVal[-1], retVal[0] - 1, -1)
 
     return retVal

lib/core/convert.py

@@ -577,7 +577,7 @@ def getUnicode(value, encoding=None, noneToNull=False):
 
         try:
             return six.text_type(value, encoding or (kb.get("pageEncoding") if kb.get("originalPage") else None) or UNICODE_ENCODING)
-        except UnicodeDecodeError:
+        except (UnicodeDecodeError, LookupError):  # LookupError: an unknown/invalid encoding name must fall back, not crash
             return six.text_type(value, UNICODE_ENCODING, errors="reversible")
     elif isListLike(value):
         value = list(getUnicode(_, encoding, noneToNull) for _ in value)

lib/core/option.py

@@ -520,7 +520,8 @@ def _setOpenApiTargets():
         checkFile(conf.openApiFile)
         infoMsg = "parsing OpenAPI/Swagger specification from '%s'" % conf.openApiFile
         logger.info(infoMsg)
-        content = openFile(conf.openApiFile).read()
+        with openFile(conf.openApiFile) as f:
+            content = f.read()
 
     tags = [_.strip() for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.openApiTags) if _.strip()] if conf.openApiTags else None
     if tags:
@@ -835,7 +836,8 @@ def _listTamperingFunctions():
         logger.info(infoMsg)
 
         for script in sorted(glob.glob(os.path.join(paths.SQLMAP_TAMPER_PATH, "*.py"))):
-            content = openFile(script, 'r').read()
+            with openFile(script, 'r') as f:
+                content = f.read()
             match = re.search(r'(?s)__priority__.+"""(.+)"""', content)
             if match:
                 comment = match.group(1).strip()
@@ -2309,6 +2311,9 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     # calibrated TRUE/FALSE reference bodies for the boolean same-HTTP-code anomaly guard (inference.py)
     kb.trueTemplate = None
     kb.falseTemplate = None
+
+    # latched once network jitter is observed, so character validation escalates to a majority vote
+    kb.jitterSeen = False
     kb.pageStable = None
     kb.pageStructurallyStable = None
     kb.partRun = None

lib/core/optiondict.py

@@ -289,7 +289,7 @@
         "noHuffman": "boolean",
         "profile": "boolean",
         "forceDns": "boolean",
-        "murphyRate": "integer",
+        "jitter": "integer",
         "smokeTest": "boolean",
         "fpTest": "boolean",
         "payloadLint": "boolean",

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.242"
+VERSION = "1.10.7.245"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -180,8 +180,26 @@
 # Identify WAF/IPS inside limited size of responses
 IDENTYWAF_PARSE_PAGE_LIMIT = 4 * 1024
 
-# Maximum sleep time in "Murphy" (testing) mode
-MAX_MURPHY_SLEEP_TIME = 3
+# Ceiling (seconds) for a simulated heavy-tailed latency spike in '--jitter' (testing) mode
+MAX_JITTER_SPIKE_TIME = 6
+
+# '--jitter=N' fault injection: ~1 in N requests is perturbed to stress the time-/boolean-based blind
+# oracles' jitter defenses. Each fired event either adds realistic response LATENCY - Gaussian jitter
+# spanning low->high network noise, or (JITTER_SPIKE_CHANCE of the time) a heavy-tailed spike - and lets
+# the genuine request proceed, or short-circuits with a transient "junk" HTTP response (gateway 5xx,
+# rate-limit, a same-HTTP-code interstitial/maintenance page, or an empty body). Values come from the
+# offline jitter studies (tests/test_jitter_stress.py, tests/test_boolean_jitter.py).
+JITTER_SIGMAS = (0.3, 0.5, 0.9)  # low / medium / high continuous jitter (seconds)
+JITTER_SPIKE_CHANCE = 0.25       # portion of latency events replaced by a heavy-tailed spike
+JITTER_JUNK_RESPONSES = (
+    ("<html><h1>502 Bad Gateway</h1></html>", 502),
+    ("<html><h1>503 Service Unavailable</h1></html>", 503),
+    ("<html><h1>504 Gateway Time-out</h1></html>", 504),
+    ('{"error": "too many requests"}', 429),
+    ("<html><head><title>Just a moment...</title></head><body>Checking your browser before accessing.</body></html>", 200),
+    ("<html><body>We'll be back shortly. Scheduled maintenance in progress.</body></html>", 200),
+    ("", 200),
+)
 
 # Regular expression used for extracting results from Google search
 GOOGLE_REGEX = r"webcache\.googleusercontent\.com/search\?q=cache:[^:]+:([^+]+)\+&amp;cd=|url\?\w+=((?![^>]+webcache\.googleusercontent\.com)http[^>]+)&(sa=U|rct=j)"
@@ -533,8 +551,11 @@
     r'"(?:errmsg|errorMessage|reason|msg)"\s*:\s*"(?P<result>[^"]+)"'      # generic JSON error-message field (NoSQL document/REST back-ends)
 )
 
-# Regular expression used for parsing charset info from meta html headers
-META_CHARSET_REGEX = r"""(?si)<head\b[^>]*>.*<meta[^>]+charset\s*=\s*["']?(?P<result>[^"'> ]+).*</head>"""
+# Regular expression used for parsing charset info from meta html headers (Note: the tempered token
+# '(?:(?!</head>).)*?' keeps the meta strictly INSIDE <head> - as the old trailing '.*</head>' did -
+# while the bounded meta-attr scan '{0,300}?' keeps it LINEAR; the old greedy form went quadratic and
+# hung for many minutes on an attacker-controlled body full of '<meta' tokens lacking '>'/'</head>')
+META_CHARSET_REGEX = r"""(?si)<head\b[^>]*>(?:(?!</head>).)*?<meta[^>]{0,300}?charset\s*=\s*["']?(?P<result>[^"'> ]+)"""
 
 # Regular expression used for parsing refresh info from meta html headers
 META_REFRESH_REGEX = r'(?i)<meta http-equiv="?refresh"?[^>]+content="?[^">]+;\s*(url=)?["\']?(?P<result>[^\'">]+)'

lib/parse/cmdline.py

@@ -887,7 +887,7 @@ def cmdLineParser(argv=None):
         parser.add_argument("--yuge", dest="yuge", action="store_true",
             help=SUPPRESS)
 
-        parser.add_argument("--murphy-rate", dest="murphyRate", type=int,
+        parser.add_argument("--jitter", dest="jitter", type=int,
             help=SUPPRESS)
 
         parser.add_argument("--debug", dest="debug", action="store_true",

lib/request/comparison.py

@@ -67,7 +67,7 @@ def comparison(page, headers, code=None, getRatioValue=False, pageLength=None):
     return _
 
 def _adjust(condition, getRatioValue):
-    if not any((conf.string, conf.notString, conf.regexp, conf.code)):
+    if not any((conf.string, conf.notString, conf.regexp, conf.code, conf.lengths)):
         # Negative logic approach is used in raw page comparison scheme as that what is "different" than original
         # PAYLOAD.WHERE.NEGATIVE response is considered as True; in switch based approach negative logic is not
         # applied as that what is by user considered as True is that what is returned by the comparison mechanism

lib/request/connect.py

@@ -113,7 +113,10 @@
 from lib.core.settings import MAX_CONNECTIONS_REGEX
 from lib.core.settings import MAX_CONNECTION_TOTAL_SIZE
 from lib.core.settings import MAX_CONSECUTIVE_CONNECTION_ERRORS
-from lib.core.settings import MAX_MURPHY_SLEEP_TIME
+from lib.core.settings import MAX_JITTER_SPIKE_TIME
+from lib.core.settings import JITTER_SIGMAS
+from lib.core.settings import JITTER_SPIKE_CHANCE
+from lib.core.settings import JITTER_JUNK_RESPONSES
 from lib.core.settings import META_REFRESH_REGEX
 from lib.core.settings import MAX_TIME_RESPONSES
 from lib.core.settings import MIN_TIME_RESPONSES
@@ -374,17 +377,31 @@ def getPage(**kwargs):
 
                     setHTTPHandlers()
 
-        if conf.dummy or conf.murphyRate and randomInt() % conf.murphyRate == 0:
-            if conf.murphyRate:
-                time.sleep(randomInt() % (MAX_MURPHY_SLEEP_TIME + 1))
-
-            page, headers, code = randomStr(int(randomInt()), alphabet=[_unichr(_) for _ in xrange(256)]), None, None if not conf.murphyRate else randomInt(3)
-
+        if conf.dummy:
+            page, headers, code = randomStr(int(randomInt()), alphabet=[_unichr(_) for _ in xrange(256)]), None, None
             threadData.lastPage = page
             threadData.lastCode = code
-
             return page, headers, code
 
+        # Simulated jitter (--jitter=N, testing): ~1 in N requests is perturbed to stress the time-/
+        # boolean-based blind jitter defenses. Half the fired events add realistic response LATENCY
+        # (Gaussian jitter across low->high noise, or an occasional heavy-tailed spike) and let the
+        # genuine request proceed; the other half short-circuit with a transient "junk" response
+        # (gateway 5xx, rate-limit, a same-HTTP-code interstitial/maintenance page, or an empty body).
+        if conf.jitter and randomInt() % conf.jitter == 0:
+            if randomInt() % 2 == 0:
+                if random.random() < JITTER_SPIKE_CHANCE:
+                    time.sleep(random.uniform(MAX_JITTER_SPIKE_TIME / 2.0, MAX_JITTER_SPIKE_TIME))  # heavy-tailed spike
+                else:
+                    time.sleep(abs(random.gauss(0, random.choice(JITTER_SIGMAS))))  # continuous jitter
+                # NOTE: falls through to the genuine request, which now carries the injected latency
+            else:
+                page, code = random.choice(JITTER_JUNK_RESPONSES)
+                headers = None
+                threadData.lastPage = page
+                threadData.lastCode = code
+                return page, headers, code
+
         if conf.liveCookies:
             with kb.locks.liveCookies:
                 if not checkFile(conf.liveCookies, raiseOnError=False) or os.path.getsize(conf.liveCookies) == 0:
@@ -406,7 +423,8 @@ def getPage(**kwargs):
                         errMsg = "problem occurred while loading cookies from file '%s'" % conf.liveCookies
                         raise SqlmapValueException(errMsg)
 
-                cookie = openFile(conf.liveCookies).read().strip()
+                with openFile(conf.liveCookies) as f:
+                    cookie = f.read().strip()
                 cookie = re.sub(r"(?i)\ACookie:\s*", "", cookie)
 
         if multipart:
@@ -545,7 +563,8 @@ def getPage(**kwargs):
                 headers = forgeHeaders(auxHeaders, headers)
 
             if kb.headersFile:
-                content = openFile(kb.headersFile, 'r').read()
+                with openFile(kb.headersFile, 'r') as f:
+                    content = f.read()
                 for line in content.split("\n"):
                     line = getText(line.strip())
                     if ':' in line:

lib/techniques/blind/inference.py

@@ -539,26 +539,35 @@ def validateChar(idx, value):
                 forgedPayload = validationPayload.replace(markingValue, unescapedCharValue)
                 forgedPayload = safeStringFormat(forgedPayload, (expressionUnescaped, idx))
 
-            result = not Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
-
-            if result and getTechniqueData() is not None:
-                trueCode, falseCode = getTechniqueData().trueCode, getTechniqueData().falseCode
-                if timeBasedCompare:
-                    if trueCode:
-                        result = threadData.lastCode == trueCode
-                        if not result:
-                            warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, trueCode)
-                            singleTimeWarnMessage(warnMsg)
-                # A boolean validation confirmed under an UNEXPECTED HTTP code (a transient 5xx/403/429/..
-                # landing on the validation request itself) is not trustworthy - fail it so the character is
-                # re-extracted, riding out the blip. On a clean target every code is true/false -> no-op.
-                elif threadData.lastCode is not None and any((trueCode, falseCode)) and threadData.lastCode not in (trueCode, falseCode):
-                    result = False
-                    singleTimeWarnMessage("unexpected HTTP code '%s' during validation phase; will re-extract" % threadData.lastCode)
+            def _check():
+                result = not Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
 
-            incrementCounter(getTechnique())
+                if result and getTechniqueData() is not None:
+                    trueCode, falseCode = getTechniqueData().trueCode, getTechniqueData().falseCode
+                    if timeBasedCompare:
+                        if trueCode:
+                            result = threadData.lastCode == trueCode
+                            if not result:
+                                warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, trueCode)
+                                singleTimeWarnMessage(warnMsg)
+                    # A boolean validation confirmed under an UNEXPECTED HTTP code (a transient 5xx/403/429/..
+                    # landing on the validation request itself) is not trustworthy - fail it so the character is
+                    # re-extracted, riding out the blip. On a clean target every code is true/false -> no-op.
+                    elif threadData.lastCode is not None and any((trueCode, falseCode)) and threadData.lastCode not in (trueCode, falseCode):
+                        result = False
+                        singleTimeWarnMessage("unexpected HTTP code '%s' during validation phase; will re-extract" % threadData.lastCode)
 
-            return result
+                incrementCounter(getTechnique())
+                return result
+
+            # Adaptive majority vote: once jitter has been observed this run a single re-check can itself
+            # be corrupted, so confirm the character by best-of-3 independent re-checks. Confidence-gated
+            # -> exact no-op (single check) on a clean run or before any jitter is seen.
+            if kb.get("jitterSeen"):
+                votes = [_check() for _ in xrange(3)]
+                return votes.count(True) >= 2
+
+            return _check()
 
         def huffmanChar(idx):
             """
@@ -823,6 +832,9 @@ def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None,
                             unexpectedResponse = True
                             singleTimeWarnMessage("unexpected response content detected. Will use (extra) validation step in similar cases")
 
+                        if unexpectedCode or unexpectedResponse:
+                            kb.jitterSeen = True   # latch: jitter observed -> escalate validateChar to a vote
+
                     if result:
                         minValue = posValue
 
@@ -862,7 +874,12 @@ def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None,
                             retVal = minValue + 1
 
                             if retVal in originalTbl or (retVal == ord('\n') and CHAR_INFERENCE_MARK in payload):
-                                if (timeBasedCompare or unexpectedCode or unexpectedResponse) and kb.get("timeless") is None and not validateChar(idx, retVal):
+                                # Once jitter has been observed this run, confirm EVERY resolved character
+                                # (via the best-of-3 vote in validateChar), not only visibly-glitched ones:
+                                # a same-HTTP-code junk that resembles a model corrupts a bit INVISIBLY, and a
+                                # single-char value (e.g. a boolean --is-dba) has no other char to later trip
+                                # detection. Confidence-gated on kb.jitterSeen -> no-op on a clean target.
+                                if (timeBasedCompare or unexpectedCode or unexpectedResponse or kb.get("jitterSeen")) and kb.get("timeless") is None and not validateChar(idx, retVal):
                                     if restricted:
                                         # the character fell outside this column's observed range - re-extract
                                         # over the full charset (not timing noise, so no delay increase / retry count)
@@ -871,7 +888,10 @@ def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None,
                                         kb.originalTimeDelay = conf.timeSec
 
                                     threadData.validationRun = 0
-                                    if (retried or 0) < MAX_REVALIDATION_STEPS:
+                                    kb.jitterSeen = True   # a needed re-extraction is itself a jitter signal (covers time-based)
+                                    # under detected jitter, allow more retries so a transient burst can't exhaust the budget
+                                    maxRevalidation = MAX_REVALIDATION_STEPS * 3 if kb.jitterSeen else MAX_REVALIDATION_STEPS
+                                    if (retried or 0) < maxRevalidation:
                                         errMsg = "invalid character detected. retrying.."
                                         logger.error(errMsg)
 

lib/utils/api.py

@@ -875,7 +875,8 @@ def download(taskid, target, filename):
 
     if os.path.isfile(path):
         logger.debug("(%s) Retrieved content of file %s" % (taskid, target))
-        content = openFile(path, "rb").read()
+        with openFile(path, "rb") as f:
+            content = f.read()
         return jsonize({"success": True, "file": encodeBase64(content, binary=False)})
     else:
         logger.warning("[%s] File does not exist %s" % (taskid, target))

lib/utils/hash.py

@@ -1117,7 +1117,7 @@ def _bruteProcessVariantA(attack_info, hash_regex, suffix, retVal, proc_id, proc
                 word = word + suffix
 
             try:
-                current = __functions__[hash_regex](password=word, uppercase=False)
+                current = __functions__[hash_regex](password=getBytes(word, unsafe=False), uppercase=False)
 
                 if current in hashes:
                     for item in attack_info[:]:
@@ -1195,7 +1195,7 @@ def _bruteProcessVariantB(user, hash_, kwargs, hash_regex, suffix, retVal, found
                 word = word + suffix
 
             try:
-                current = __functions__[hash_regex](password=word, uppercase=False, **kwargs)
+                current = __functions__[hash_regex](password=getBytes(word, unsafe=False), uppercase=False, **kwargs)
 
                 if hash_ == current:
                     if hash_regex == HASH.ORACLE_OLD:  # only for cosmetic purposes
@@ -1285,7 +1285,7 @@ def _bruteProcessVariantSalted(attack_info, hash_regex, suffix, retVal, proc_id,
                 ((user, hash_), kwargs) = item
 
                 try:
-                    current = __functions__[hash_regex](password=word, uppercase=False, **kwargs)
+                    current = __functions__[hash_regex](password=getBytes(word, unsafe=False), uppercase=False, **kwargs)
 
                     if hash_ == current:
                         retVal.put((user, hash_, word))