Fix support for BlueCoat reputation checking

README.md

@@ -8,6 +8,11 @@ This Python based tool was written to quickly query the Expireddomains.net searc
 
 ## Changes
 
+- 15 May 2020
+   + Fix BlueCoat reputation checking to bypass XSRF and additional POST paramater checks
+   + Add internal code comments for readability
+   + Add check for ExpiredDomains username before asking for a password
+  
 - 21 February 2020
    + updated Pillow version to support Python3.7+
    + Add instructions to install using pipenv

domainhunter.py

@@ -18,13 +18,14 @@ import os
 import sys
 from urllib.parse import urlparse
 import getpass
-import uuid
 
-__version__ = "20190716"
+__version__ = "20200515"
 
 ## Functions
 
 def doSleep(timing):
+    """Add nmap like random sleep interval for multiple requests"""
+
     if timing == 0:
         time.sleep(random.randrange(90,120))
     elif timing == 1:
@@ -38,6 +39,8 @@ def doSleep(timing):
     # There's no elif timing == 5 here because we don't want to sleep for -t 5
 
 def checkUmbrella(domain):
+    """Umbrella Domain reputation service"""
+
     try:
         url = 'https://investigate.api.umbrella.com/domains/categorization/?showLabels'
         postData = [domain]
@@ -61,13 +64,41 @@ def checkUmbrella(domain):
         print('[-] Error retrieving Umbrella reputation! {0}'.format(e))
         return "error"
 
-
 def checkBluecoat(domain):
+    """Symantec Sitereview Domain Reputation"""
+
     try:
-        url = 'https://sitereview.bluecoat.com/resource/lookup'
-        postData = {'url':domain,'captcha':''}
+        headers = {
+            'User-Agent':useragent,
+            'Referer':'http://sitereview.bluecoat.com/'}
 
-        token = str(uuid.uuid4())
+        # Establish our session information
+        response = s.get("https://sitereview.bluecoat.com/",headers=headers,verify=False,proxies=proxies)
+        response = s.head("https://sitereview.bluecoat.com/resource/captcha-request",headers=headers,verify=False,proxies=proxies)
+        
+        # Pull the XSRF Token from the cookie jar
+        session_cookies = s.cookies.get_dict()
+        if "XSRF-TOKEN" in session_cookies:
+            token = session_cookies["XSRF-TOKEN"]
+        else:
+            raise NameError("No XSRF-TOKEN found in the cookie jar")
+ 
+        # Perform SiteReview lookup
+        
+        # BlueCoat Added base64 encoded phrases selected at random and sha256 hashing of the JSESSIONID
+        phrases = [
+            'UGxlYXNlIGRvbid0IGZvcmNlIHVzIHRvIHRha2UgbWVhc3VyZXMgdGhhdCB3aWxsIG1ha2UgaXQgbW9yZSBkaWZmaWN1bHQgZm9yIGxlZ2l0aW1hdGUgdXNlcnMgdG8gbGV2ZXJhZ2UgdGhpcyBzZXJ2aWNlLg==',
+            'SWYgeW91IGNhbiByZWFkIHRoaXMsIHlvdSBhcmUgbGlrZWx5IGFib3V0IHRvIGRvIHNvbWV0aGluZyB0aGF0IGlzIGFnYWluc3Qgb3VyIFRlcm1zIG9mIFNlcnZpY2U=',
+            'RXZlbiBpZiB5b3UgYXJlIG5vdCBwYXJ0IG9mIGEgY29tbWVyY2lhbCBvcmdhbml6YXRpb24sIHNjcmlwdGluZyBhZ2FpbnN0IFNpdGUgUmV2aWV3IGlzIHN0aWxsIGFnYWluc3QgdGhlIFRlcm1zIG9mIFNlcnZpY2U=',
+            'U2NyaXB0aW5nIGFnYWluc3QgU2l0ZSBSZXZpZXcgaXMgYWdhaW5zdCB0aGUgU2l0ZSBSZXZpZXcgVGVybXMgb2YgU2VydmljZQ=='
+        ]
+        
+        postData = {
+            'url':domain,
+            'captcha':'',
+            'key':'%032x' % random.getrandbits(256), # Generate a random 256bit "hash-like" string
+            'phrase':random.choice(phrases), # Pick a random base64 phrase from the list
+            'source':'new-lookup'}
 
         headers = {'User-Agent':useragent,
                    'Accept':'application/json, text/plain, */*',
@@ -76,52 +107,50 @@ def checkBluecoat(domain):
                    'X-XSRF-TOKEN':token,
                    'Referer':'http://sitereview.bluecoat.com/'}
 
-        c = {
-            "JSESSIONID":str(uuid.uuid4()).upper().replace("-", ""),
-            "XSRF-TOKEN":token
-        }
-
         print('[*] BlueCoat: {}'.format(domain))
+        response = s.post('https://sitereview.bluecoat.com/resource/lookup',headers=headers,json=postData,verify=False,proxies=proxies)
         
-        response = s.post(url,headers=headers,cookies=c,json=postData,verify=False,proxies=proxies)
-        responseJSON = json.loads(response.text)
-        
-        if 'errorType' in responseJSON:
-            a = responseJSON['errorType']
+        # Check for any HTTP errors
+        if response.status_code != 200:
+            a = "HTTP Error ({}-{}) - Is your IP blocked?".format(response.status_code,response.reason)
         else:
-            a = responseJSON['categorization'][0]['name']
+            responseJSON = json.loads(response.text)
         
-        # Print notice if CAPTCHAs are blocking accurate results and attempt to solve if --ocr
-        if a == 'captcha':
-            if ocr:
-                # This request is also performed by a browser, but is not needed for our purposes
-                #captcharequestURL = 'https://sitereview.bluecoat.com/resource/captcha-request'
-
-                print('[*] Received CAPTCHA challenge!')
-                captcha = solveCaptcha('https://sitereview.bluecoat.com/resource/captcha.jpg',s)
-                
-                if captcha:
-                    b64captcha = base64.urlsafe_b64encode(captcha.encode('utf-8')).decode('utf-8')
-                   
-                    # Send CAPTCHA solution via GET since inclusion with the domain categorization request doens't work anymore
-                    captchasolutionURL = 'https://sitereview.bluecoat.com/resource/captcha-request/{0}'.format(b64captcha)
-                    print('[*] Submiting CAPTCHA at {0}'.format(captchasolutionURL))
-                    response = s.get(url=captchasolutionURL,headers=headers,verify=False,proxies=proxies)
+            if 'errorType' in responseJSON:
+                a = responseJSON['errorType']
+            else:
+                a = responseJSON['categorization'][0]['name']
+        
+            # Print notice if CAPTCHAs are blocking accurate results and attempt to solve if --ocr
+            if a == 'captcha':
+                if ocr:
+                    # This request is also performed by a browser, but is not needed for our purposes
+                    #captcharequestURL = 'https://sitereview.bluecoat.com/resource/captcha-request'
+
+                    print('[*] Received CAPTCHA challenge!')
+                    captcha = solveCaptcha('https://sitereview.bluecoat.com/resource/captcha.jpg',s)
+                    
+                    if captcha:
+                        b64captcha = base64.urlsafe_b64encode(captcha.encode('utf-8')).decode('utf-8')
+                    
+                        # Send CAPTCHA solution via GET since inclusion with the domain categorization request doens't work anymore
+                        captchasolutionURL = 'https://sitereview.bluecoat.com/resource/captcha-request/{0}'.format(b64captcha)
+                        print('[*] Submiting CAPTCHA at {0}'.format(captchasolutionURL))
+                        response = s.get(url=captchasolutionURL,headers=headers,verify=False,proxies=proxies)
 
-                    # Try the categorization request again
-                    response = s.post(url,headers=headers,cookies=c,json=postData,verify=False,proxies=proxies)
+                        # Try the categorization request again
+                        response = s.post(url,headers=headers,cookies=c,json=postData,verify=False,proxies=proxies)
 
-                    responseJSON = json.loads(response.text)
+                        responseJSON = json.loads(response.text)
 
-                    if 'errorType' in responseJSON:
-                        a = responseJSON['errorType']
+                        if 'errorType' in responseJSON:
+                            a = responseJSON['errorType']
+                        else:
+                            a = responseJSON['categorization'][0]['name']
                     else:
-                        a = responseJSON['categorization'][0]['name']
+                        print('[-] Error: Failed to solve BlueCoat CAPTCHA with OCR! Manually solve at "https://sitereview.bluecoat.com/sitereview.jsp"')
                 else:
-                    print('[-] Error: Failed to solve BlueCoat CAPTCHA with OCR! Manually solve at "https://sitereview.bluecoat.com/sitereview.jsp"')
-            else:
-                print('[-] Error: BlueCoat CAPTCHA received. Try --ocr flag or manually solve a CAPTCHA at "https://sitereview.bluecoat.com/sitereview.jsp"')
-
+                    print('[-] Error: BlueCoat CAPTCHA received. Try --ocr flag or manually solve a CAPTCHA at "https://sitereview.bluecoat.com/sitereview.jsp"')
         return a
 
     except Exception as e:
@@ -129,6 +158,8 @@ def checkBluecoat(domain):
         return "error"
 
 def checkIBMXForce(domain):
+    """IBM XForce Domain Reputation"""
+
     try: 
         url = 'https://exchange.xforce.ibmcloud.com/url/{}'.format(domain)
         headers = {'User-Agent':useragent,
@@ -155,7 +186,7 @@ def checkIBMXForce(domain):
         else:
             categories = ''
             # Parse all dictionary keys and append to single string to get Category names
-            for key in responseJSON["result"]['cats']:
+            for key in responseJSON['result']['cats']:
                 categories += '{0}, '.format(str(key))
 
             a = '{0}(Score: {1})'.format(categories,str(responseJSON['result']['score']))
@@ -167,6 +198,8 @@ def checkIBMXForce(domain):
         return "error"
 
 def checkTalos(domain):
+    """Cisco Talos Domain Reputation"""
+
     url = 'https://www.talosintelligence.com/sb_api/query_lookup?query=%2Fapi%2Fv2%2Fdetails%2Fdomain%2F&query_entry={0}&offset=0&order=ip+asc'.format(domain)
     headers = {'User-Agent':useragent,
                'Referer':url}
@@ -195,6 +228,7 @@ def checkTalos(domain):
         return "error"
 
 def checkMXToolbox(domain):
+    """ Checks the MXToolbox service for Google SafeBrowsing and PhishTank information. Currently broken"""
     url = 'https://mxtoolbox.com/Public/Tools/BrandReputation.aspx'
     headers = {'User-Agent':useragent,
             'Origin':url,
@@ -254,6 +288,8 @@ def checkMXToolbox(domain):
         return "error"
 
 def downloadMalwareDomains(malwaredomainsURL):
+    """Downloads a current list of known malicious domains"""
+
     url = malwaredomainsURL
     response = s.get(url=url,headers=headers,verify=False,proxies=proxies)
     responseText = response.text
@@ -263,6 +299,8 @@ def downloadMalwareDomains(malwaredomainsURL):
         print("[-] Error reaching:{}  Status: {}").format(url, response.status_code)
 
 def checkDomain(domain):
+    """Executes various domain reputation checks included in the project"""
+
     print('[*] Fetching domain reputation for: {}'.format(domain))
 
     if domain in maldomainsList:
@@ -277,8 +315,10 @@ def checkDomain(domain):
     ciscotalos = checkTalos(domain)
     print("[+] {}: {}".format(domain, ciscotalos))
 
-    mxtoolbox = checkMXToolbox(domain)
-    print("[+] {}: {}".format(domain, mxtoolbox))
+    #This service has completely changed, removing for now
+    #mxtoolbox = checkMXToolbox(domain)
+    #print("[+] {}: {}".format(domain, mxtoolbox))
+    mxtoolbox = "-"
 
     umbrella = "not available"
     if len(umbrella_apikey):
@@ -291,8 +331,9 @@ def checkDomain(domain):
     return results
 
 def solveCaptcha(url,session):  
-    # Downloads CAPTCHA image and saves to current directory for OCR with tesseract
-    # Returns CAPTCHA string or False if error occured
+    """Downloads CAPTCHA image and saves to current directory for OCR with tesseract
+    Returns CAPTCHA string or False if error occured
+    """
     
     jpeg = 'captcha.jpg'
     
@@ -324,7 +365,7 @@ def solveCaptcha(url,session):
         return False
 
 def drawTable(header,data):
-    
+    """Generates a text based table for printing to the console"""
     data.insert(0,header)
     t = Texttable(max_width=maxwidth)
     t.add_rows(data)
@@ -332,7 +373,9 @@ def drawTable(header,data):
     
     return(t.draw())
 
-def loginExpiredDomains():    
+def loginExpiredDomains():
+    """Login to the ExpiredDomains site with supplied credentials"""
+
     data = "login=%s&password=%s&redirect_2_url=/" % (username, password)
     headers["Content-Type"] = "application/x-www-form-urlencoded"
     r = s.post(expireddomainHost + "/login/", headers=headers, data=data, proxies=proxies, verify=False, allow_redirects=False)
@@ -449,14 +492,13 @@ Examples:
     umbrella_apikey = args.umbrella_apikey
 
     malwaredomainsURL = 'http://mirror1.malwaredomains.com/files/justdomains'
-
     expireddomainsqueryURL = 'https://www.expireddomains.net/domain-name-search'
     expireddomainHost = "https://member.expireddomains.net"
 
     timestamp = time.strftime("%Y%m%d_%H%M%S")
             
     useragent = 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)'
-   
+
     headers = {'User-Agent':useragent}
 
     proxies = {}
@@ -470,7 +512,7 @@ Examples:
         proxy_parts = urlparse(args.proxy)
         proxies["http"] = "http://%s" % (proxy_parts.netloc)
         proxies["https"] = "https://%s" % (proxy_parts.netloc)
-
+    s.proxies = proxies
     title = '''
  ____   ___  __  __    _    ___ _   _   _   _ _   _ _   _ _____ _____ ____  
 |  _ \ / _ \|  \/  |  / \  |_ _| \ | | | | | | | | | \ | |_   _| ____|  _ \ 
@@ -527,7 +569,9 @@ If you plan to use this content for illegal purpose, don't.  Have a nice day :)'
 
     # Generate list of URLs to query for expired/deleted domains
     urls = []
-    
+    if username == None or username == "":
+        print('[-] Error: ExpiredDomains.net requires a username! Use the --username parameter')
+        exit(1)
     if args.password == None or args.password == "":
         password = getpass.getpass("expireddomains.net Password: ")
 

requirements.txt

@@ -4,5 +4,4 @@ beautifulsoup4==4.5.3
 lxml
 pillow==5.4.0
 pytesseract
-urllib3
-uuid
+urllib3