Entropy calculation

CHANGES.md

@@ -0,0 +1,31 @@
+## Main feature
+
+This module was created to roughly calculate randomness in redownloaded pictures.
+
+It can be used to detect possible memory leaks from image renderers.
+If there is some error in the image renderer logic, and it is not handling exceptions properly,
+it is possible that a malformed uploaded file will trigger it, and the response will contain parts of the 
+image renderer memory.
+This behaviour will increase the randomness of pixels in the returned picture.
+
+By getting the full RGB list from the response picture, and compressing it with zlib, we can
+estimate the randomness of the pixels by dividing these two values with each other.
+
+This module will give an informational issue to all redownloaded pictures, where the compression ratio is beneath the
+median of all compression ratios, which can help us to see, if there is too much random noise in the picture,
+and from there, we can manually investigate the incident.
+
+## When/How it is used
+
+Image entropy calculation is only done on the Fingerping, fuzzer, and recursive uploader module, if the "Calculate image entropy" checkbox is selected. 
+The module will create informational issues where the redownloaded picture compression ratio is lower than the avarage of it. It can be used to detect issues such as this: https://blog.silentsignal.eu/2019/04/18/drop-by-drop-bleeding-through-libvips/
+
+## Changes in the code:
+
+Added a checkbox to the general options UI with label "Calculate image entropy". It's referenced in the code as calculate_entropy
+
+Added new functions for the  entropy calculation/reporting. These are defined from line 866 to 922.
+
+The calculation is called in the affected modules(Fingerping/Fuzzer/Recursive uploader), where all the ratios and request/responses are stored in a local variable called Entropy_list, which is used at the end of each module for the reporting.
+
+Added a new helper function which calculates the median of a given list. (line 4243)

UploadScanner.py

@@ -862,6 +862,66 @@ def add_log_entry(self, rr):
                                self._helpers.analyzeRequest(rr).getUrl()))
             self.fireTableRowsInserted(row, row)
 
+    # Helper functions for image entropy calculation
+    def _calculate_image_entropy_wrapper(self, injector, urr):
+        download_responseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse())
+        headers = [FloydsHelpers.u2s(x) for x in download_responseInfo.getHeaders()]
+        if any("image/" in h for h in headers):
+            ratio = self._calculate_image_entropy(injector, urr)
+            return ratio
+
+    def _report_image_entropy(self, Entropy_list):
+        # Calculate median of the ratios
+        ratios = [ratio[0] for ratio in Entropy_list]
+        median = self._median(ratios)
+        for item in Entropy_list:
+            if item[0] and item[0] < median:
+                injector = item[1]
+                urr = item[2]
+                ratio = item[0]
+                service = urr.download_rr.getHttpService()
+                url = self._helpers.analyzeRequest(urr.download_rr).getUrl()
+                name = "Downloaded image entropy info"
+                severity = "Information"
+                confidence = "Firm"
+                detail = "Information about the redownloaded image entropy where compression ratios were lower than the avarage. <br> " \
+                     "If the compression ratio is too low, that could mean that the image contains memory leak.<br><br> " \
+                     "The avarage compression ratio for the module was:<b> {} </b><br><br> " \
+                     "Compression ratio for this image:<b> {} </b><br>".format(median,ratio)
+
+                csi = CustomScanIssue([injector.get_brr()], name, detail, confidence, severity, service, url)
+                csi.httpMessagesPy = [urr.upload_rr, urr.download_rr]
+                if csi:
+                    self._callbacks.addScanIssue(csi)
+
+
+
+    # Function to calculate image entropy via compressing redownloaded images
+    def _calculate_image_entropy(self, injector, urr):
+        download_iResponseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse())
+        download_body = FloydsHelpers.jb2ps(urr.download_rr.getResponse())[download_iResponseInfo.getBodyOffset():]
+
+        try:
+            # Rescaling the image for performance            
+            picture_width, picture_height, fileformat = ImageHelpers.image_width_height(download_body)
+            if picture_width and picture_height and fileformat:
+                if picture_width >= 200 or picture_height >= 200:
+                    download_thumbnail = ImageHelpers.rescale_image(200, 200, download_body)
+                    if download_thumbnail:
+                        download_body = download_thumbnail
+
+            download_rgbs = ImageHelpers.get_image_rgb_list(download_body)
+            download_conv = "".join([struct.pack("<I",i) for i in download_rgbs])
+            download_compressed = zlib.compress(download_conv, 9)
+
+            ratio = len(download_rgbs)*4/(len(download_compressed)*1.0)
+
+            return ratio
+
+        except:
+            return None
+
+
     # Implement IHttpListener
     def processHttpMessage(self, _, messageIsRequest, base_request_response):
         try:
@@ -890,6 +950,7 @@ def processHttpMessage(self, _, messageIsRequest, base_request_response):
                 # ... do not scan things that are not "in scope" (see DownloadMatcherCollection class)
                 # means we only check if we uploaded stuff to that host or the user configured
                 # another host in the ReDownloader options that is therefore also "in scope"
+
                 matchers = self.dl_matchers.get_matchers_for_url(url)
                 if not matchers:
                     #We hit this for all not "in scope" requests
@@ -1191,7 +1252,7 @@ def do_checks(self, injector):
 
         # Just to make sure (maybe we write a new module above and forget this call):
         self.collab_monitor_thread.add_or_update(burp_colab, colab_tests)
-
+                
         # DoSing the server is best done at the end when we already know about everything else...
         # Timeout and DoS - generic
         if not scan_was_stopped:
@@ -3343,6 +3404,7 @@ def _polyglot(self, injector, burp_colab):
         return colab_tests
 
     def _fingerping(self, injector):
+        Entropy_list = []
         if not injector.opts.file_formats['png'].isSelected():
             # we only upload PNG files in this module
             return
@@ -3372,6 +3434,13 @@ def _fingerping(self, injector):
                     resp = FloydsHelpers.jb2ps(urr.download_rr.getResponse())
                     body_offset = i_response_info.getBodyOffset()
                     body = resp[body_offset:]
+
+                    # Calculate image entropy
+                    if injector.opts.calculate_entropy and urr and urr.download_rr:
+                        ratio = self._calculate_image_entropy_wrapper(injector, urr)
+                        if ratio:
+                            Entropy_list.append([ratio, injector, urr])
+
                     if body.startswith('\x89PNG'):
                         # print "Downloaded", orig_filename, "is a PNG. Content:"
                         # print repr(body)
@@ -3432,6 +3501,12 @@ def _fingerping(self, injector):
         issue = self._create_issue_template(injector.get_brr(), title, desc, confidence, "Information")
         self._add_scan_issue(issue)
 
+        # Reportin image entropy results
+        if injector.opts.calculate_entropy:
+            print "Reporting image entropies for fingerping..."
+            self._report_image_entropy(Entropy_list)
+
+
 
     def _quirks_with_passive(self, injector):
         if not injector.get_uploaded_filename():
@@ -3631,6 +3706,10 @@ def _recursive_upload_files(self, injector, burp_colab):
         # file extension: from original request, from file, using the default, guessing by mime type
         # mime_type:      from original request, guessing by file, guessing by file extension
         # File content:   Always from file
+
+        # List for entropy calc
+        Entropy_list = []
+
         if not injector.opts.ru_dirpath:
             return
 
@@ -3683,7 +3762,13 @@ def _recursive_upload_files(self, injector, burp_colab):
                 print "Recursive Uploader doing", new_filename, mime_type
                 req = injector.get_request(new_filename, content, mime_type)
                 if req:
-                    self._make_http_request(injector, req)
+                    urr = self._make_http_request(injector, req, redownload_filename=new_filename)
+
+                    # Image entropy calculation
+                    if injector.opts.calculate_entropy and urr and urr.download_rr:
+                        ratio = self._calculate_image_entropy_wrapper(injector, urr)
+                        if ratio:
+                            Entropy_list.append([ratio, injector, urr])
 
                 # Combine with replacer
                 if injector.opts.ru_combine_with_replacer and burp_colab:
@@ -3699,9 +3784,17 @@ def _recursive_upload_files(self, injector, burp_colab):
                                     urr = self._make_http_request(injector, req)
                                     if urr:
                                         colab_tests.append(ColabTest(colab_url, urr, issue))
+
+        if injector.opts.calculate_entropy:
+            print "Reporting image entropies for recursive uploader..."
+            self._report_image_entropy(Entropy_list)
+
         return colab_tests
 
     def _fuzz(self, injector):
+        # Variable for image entropy calc
+        Entropy_list = []
+
         content = injector.get_uploaded_content()
         if not content:
             return
@@ -3717,7 +3810,13 @@ def _fuzz(self, injector):
             name_increment += 1
             req = injector.get_request(new_filename, new_content)
             if req:
-                self._make_http_request(injector, req)
+                urr = self._make_http_request(injector, req, redownload_filename=new_filename)            
+                # Image entropy calculation
+                if injector.opts.calculate_entropy and urr and urr.download_rr:
+                    ratio = self._calculate_image_entropy_wrapper(injector, urr)
+                    if ratio:
+                        Entropy_list.append([ratio, injector, urr])
+
         for _ in xrange(0, injector.opts.fuzzer_random_mutations):
             new_content = copy.copy(content)
             index = random.randint(0, len(new_content) - 1)
@@ -3736,7 +3835,16 @@ def _fuzz(self, injector):
             name_increment += 1
             req = injector.get_request(new_filename, new_content)
             if req:
-                self._make_http_request(injector, req)
+                urr = self._make_http_request(injector, req, redownload_filename=new_filename)
+                # Image entropy calculation
+                if injector.opts.calculate_entropy and urr and urr.download_rr:
+                    ratio = self._calculate_image_entropy_wrapper(injector, urr)
+                    if ratio:
+                        Entropy_list.append([ratio, injector, urr])
+        # Reporting image entropy
+        if injector.opts.calculate_entropy:
+            print "Reporting image entropies for fuzzer..."
+            self._report_image_entropy(Entropy_list)
 
     def _timeout_and_dos(self, injector):
         orig_filename = injector.get_uploaded_filename()
@@ -4130,6 +4238,17 @@ def _timeout_and_dos(self, injector):
                     csi = self._create_issue_template(brr, title, desc, "Tentative", "Medium")
                     self._add_scan_issue(csi)
 
+
+    # Helper function to calculate median
+    def _median(self, lst):
+        n = len(lst)
+        if n < 1:
+            return None
+        if n % 2 == 1:
+            return sorted(lst)[n//2]
+        else:
+            return sum(sorted(lst)[n//2-1:n//2+1])/2.0
+
     # Helper functions
     def _filename_to_expected(self, filename):
         # TODO feature: maybe try to download both?
@@ -4388,6 +4507,7 @@ def _make_http_request(self, injector, req, report_timeouts=True, throttle=True,
                 preflight_rr, download_rr = injector.opts.redownloader_try_redownload(resp, redownload_filename)
                 urr.preflight_rr = preflight_rr
                 urr.download_rr = download_rr
+
                 if injector.opts.create_log:
                     # create a new log entry with the message details
                     if urr.preflight_rr:
@@ -8418,6 +8538,7 @@ def __init__(self, burp_extender, callbacks, helpers, scan_controler=None, globa
         # Options general:
         self.throttle_time = 0.0
         self.sleep_time = 6.0
+        self.calculate_entropy = False
         self.create_log = False
         self.replace_filename = True
         self.replace_ct = True
@@ -8497,6 +8618,7 @@ def serialize(self):
 
         serialized_object['throttle_time'] = self.throttle_time
         serialized_object['sleep_time'] = self.sleep_time
+        serialized_object['calculate_entropy'] = self.calculate_entropy
         serialized_object['create_log'] = self.create_log
         serialized_object['replace_filename'] = self.replace_filename
         serialized_object['replace_ct'] = self.replace_ct
@@ -8565,6 +8687,8 @@ def deserialize(self, serialized_object, global_to_tab=False):
         # This "if" is necessary to be backward compatible (the old serialized object does not have this attribute)
         if 'sleep_time' in serialized_object:
             self.tf_sleep_time.setText(str(serialized_object['sleep_time']))
+        if 'calculate_entropy' in serialized_object:
+            self.cb_calculate_entropy.setSelected(serialized_object['calculate_entropy'])
         self.cb_create_log.setSelected(serialized_object['create_log'])
         self.cb_replace_filename.setSelected(serialized_object['replace_filename'])
         self.cb_replace_ct.setSelected(serialized_object['replace_ct'])
@@ -8764,6 +8888,7 @@ def create_options(self):
                                                                                 text=self.image_exiftool)
         self.lbl_throttle_time, self.tf_throttle_time = self.small_tf("Throttle between requests in seconds:", str(self.throttle_time))
         self.lbl_sleep_time, self.tf_sleep_time = self.small_tf("Sleep time for sleep payloads in seconds:", str(self.sleep_time))
+        _, self.cb_calculate_entropy = self.checkbox('Calculate entropy for images:', self.calculate_entropy)
         _, self.cb_create_log = self.checkbox('Create log, see "Done uploads" tab:', self.create_log)
         _, self.cb_replace_filename = self.checkbox('Replace filename in requests:', self.replace_filename)
         _, self.cb_replace_ct = self.checkbox('Replace content type in requests:', self.replace_ct)
@@ -8906,7 +9031,8 @@ def insertUpdate(self, _):
         except ValueError:
             self.sleep_time = 6.0
             OptionsPanel.mark_misconfigured(self.lbl_sleep_time)
-
+
+        self.calculate_entropy = self.cb_calculate_entropy.isSelected()
         self.create_log = self.cb_create_log.isSelected()
         self.replace_filename = self.cb_replace_filename.isSelected()
         self.replace_ct = self.cb_replace_ct.isSelected()