Fix Site Review, Disable MxToolbox, Add -q/quiet option (#31)

* Fix support for BlueCoat reputation checking

* Fix Symantec Site Review (Bluecoat) reputation checking
   + Temporary fix for broken malware domains link. This service is no longer offered in the form used by DomainHunter.
   + Add internal code comments for readability
   + Add check for ExpiredDomains username before asking for a password

* Add -q/quiet option and minor cleanup

* Fix linting error, undefined variable

README.md

@@ -4,9 +4,16 @@ Authors Joe Vest (@joevest) & Andrew Chiles (@andrewchiles)
 
 Domain name selection is an important aspect of preparation for penetration tests and especially Red Team engagements. Commonly, domains that were used previously for benign purposes and were properly categorized can be purchased for only a few dollars. Such domains can allow a team to bypass reputation based web filters and network egress restrictions for phishing and C2 related tasks. 
 
-This Python based tool was written to quickly query the Expireddomains.net search engine for expired/available domains with a previous history of use. It then optionally queries for domain reputation against services like Symantec WebPulse (BlueCoat), IBM X-Force, and Cisco Talos. The primary tool output is a timestamped HTML table style report.
+This Python based tool was written to quickly query the Expireddomains.net search engine for expired/available domains with a previous history of use. It then optionally queries for domain reputation against services like Symantec Site Review (BlueCoat), IBM X-Force, and Cisco Talos. The primary tool output is a timestamped HTML table style report.
 
-## Changes
+## Changelog
+
+- 07 January 2021
+   + Fix Symantec Site Review (Bluecoat) reputation checking to bypass XSRF and additional POST parameter checks
+   + Temporary fix for broken malware domains link. This service is no longer offered in the form used by DomainHunter.
+   + Add internal code comments for readability
+   + Add check for ExpiredDomains username before asking for a password
+   + Disable Google Safe Browsing/PhishTank reputation from MxToolbox as this service has changed
 
 - 21 February 2020
    + updated Pillow version to support Python3.7+

domainhunter.py

@@ -18,13 +18,14 @@ import os
 import sys
 from urllib.parse import urlparse
 import getpass
-import uuid
 
-__version__ = "20190716"
+__version__ = "20210107"
 
 ## Functions
 
 def doSleep(timing):
+    """Add nmap like random sleep interval for multiple requests"""
+
     if timing == 0:
         time.sleep(random.randrange(90,120))
     elif timing == 1:
@@ -38,6 +39,8 @@ def doSleep(timing):
     # There's no elif timing == 5 here because we don't want to sleep for -t 5
 
 def checkUmbrella(domain):
+    """Umbrella Domain reputation service"""
+
     try:
         url = 'https://investigate.api.umbrella.com/domains/categorization/?showLabels'
         postData = [domain]
@@ -61,13 +64,41 @@ def checkUmbrella(domain):
         print('[-] Error retrieving Umbrella reputation! {0}'.format(e))
         return "error"
 
-
 def checkBluecoat(domain):
+    """Symantec Sitereview Domain Reputation"""
+
     try:
-        url = 'https://sitereview.bluecoat.com/resource/lookup'
-        postData = {'url':domain,'captcha':''}
+        headers = {
+            'User-Agent':useragent,
+            'Referer':'http://sitereview.bluecoat.com/'}
 
-        token = str(uuid.uuid4())
+        # Establish our session information
+        response = s.get("https://sitereview.bluecoat.com/",headers=headers,verify=False,proxies=proxies)
+        response = s.head("https://sitereview.bluecoat.com/resource/captcha-request",headers=headers,verify=False,proxies=proxies)
+        
+        # Pull the XSRF Token from the cookie jar
+        session_cookies = s.cookies.get_dict()
+        if "XSRF-TOKEN" in session_cookies:
+            token = session_cookies["XSRF-TOKEN"]
+        else:
+            raise NameError("No XSRF-TOKEN found in the cookie jar")
+ 
+        # Perform SiteReview lookup
+        
+        # BlueCoat Added base64 encoded phrases selected at random and sha256 hashing of the JSESSIONID
+        phrases = [
+            'UGxlYXNlIGRvbid0IGZvcmNlIHVzIHRvIHRha2UgbWVhc3VyZXMgdGhhdCB3aWxsIG1ha2UgaXQgbW9yZSBkaWZmaWN1bHQgZm9yIGxlZ2l0aW1hdGUgdXNlcnMgdG8gbGV2ZXJhZ2UgdGhpcyBzZXJ2aWNlLg==',
+            'SWYgeW91IGNhbiByZWFkIHRoaXMsIHlvdSBhcmUgbGlrZWx5IGFib3V0IHRvIGRvIHNvbWV0aGluZyB0aGF0IGlzIGFnYWluc3Qgb3VyIFRlcm1zIG9mIFNlcnZpY2U=',
+            'RXZlbiBpZiB5b3UgYXJlIG5vdCBwYXJ0IG9mIGEgY29tbWVyY2lhbCBvcmdhbml6YXRpb24sIHNjcmlwdGluZyBhZ2FpbnN0IFNpdGUgUmV2aWV3IGlzIHN0aWxsIGFnYWluc3QgdGhlIFRlcm1zIG9mIFNlcnZpY2U=',
+            'U2NyaXB0aW5nIGFnYWluc3QgU2l0ZSBSZXZpZXcgaXMgYWdhaW5zdCB0aGUgU2l0ZSBSZXZpZXcgVGVybXMgb2YgU2VydmljZQ=='
+        ]
+        
+        postData = {
+            'url':domain,
+            'captcha':'',
+            'key':'%032x' % random.getrandbits(256), # Generate a random 256bit "hash-like" string
+            'phrase':random.choice(phrases), # Pick a random base64 phrase from the list
+            'source':'new-lookup'}
 
         headers = {'User-Agent':useragent,
                    'Accept':'application/json, text/plain, */*',
@@ -76,52 +107,50 @@ def checkBluecoat(domain):
                    'X-XSRF-TOKEN':token,
                    'Referer':'http://sitereview.bluecoat.com/'}
 
-        c = {
-            "JSESSIONID":str(uuid.uuid4()).upper().replace("-", ""),
-            "XSRF-TOKEN":token
-        }
-
         print('[*] BlueCoat: {}'.format(domain))
+        response = s.post('https://sitereview.bluecoat.com/resource/lookup',headers=headers,json=postData,verify=False,proxies=proxies)
         
-        response = s.post(url,headers=headers,cookies=c,json=postData,verify=False,proxies=proxies)
-        responseJSON = json.loads(response.text)
-        
-        if 'errorType' in responseJSON:
-            a = responseJSON['errorType']
+        # Check for any HTTP errors
+        if response.status_code != 200:
+            a = "HTTP Error ({}-{}) - Is your IP blocked?".format(response.status_code,response.reason)
         else:
-            a = responseJSON['categorization'][0]['name']
+            responseJSON = json.loads(response.text)
         
-        # Print notice if CAPTCHAs are blocking accurate results and attempt to solve if --ocr
-        if a == 'captcha':
-            if ocr:
-                # This request is also performed by a browser, but is not needed for our purposes
-                #captcharequestURL = 'https://sitereview.bluecoat.com/resource/captcha-request'
-
-                print('[*] Received CAPTCHA challenge!')
-                captcha = solveCaptcha('https://sitereview.bluecoat.com/resource/captcha.jpg',s)
-                
-                if captcha:
-                    b64captcha = base64.urlsafe_b64encode(captcha.encode('utf-8')).decode('utf-8')
-                   
-                    # Send CAPTCHA solution via GET since inclusion with the domain categorization request doens't work anymore
-                    captchasolutionURL = 'https://sitereview.bluecoat.com/resource/captcha-request/{0}'.format(b64captcha)
-                    print('[*] Submiting CAPTCHA at {0}'.format(captchasolutionURL))
-                    response = s.get(url=captchasolutionURL,headers=headers,verify=False,proxies=proxies)
+            if 'errorType' in responseJSON:
+                a = responseJSON['errorType']
+            else:
+                a = responseJSON['categorization'][0]['name']
+        
+            # Print notice if CAPTCHAs are blocking accurate results and attempt to solve if --ocr
+            if a == 'captcha':
+                if ocr:
+                    # This request is also performed by a browser, but is not needed for our purposes
+                    #captcharequestURL = 'https://sitereview.bluecoat.com/resource/captcha-request'
+
+                    print('[*] Received CAPTCHA challenge!')
+                    captcha = solveCaptcha('https://sitereview.bluecoat.com/resource/captcha.jpg',s)
+                    
+                    if captcha:
+                        b64captcha = base64.urlsafe_b64encode(captcha.encode('utf-8')).decode('utf-8')
+                    
+                        # Send CAPTCHA solution via GET since inclusion with the domain categorization request doesn't work anymore
+                        captchasolutionURL = 'https://sitereview.bluecoat.com/resource/captcha-request/{0}'.format(b64captcha)
+                        print('[*] Submiting CAPTCHA at {0}'.format(captchasolutionURL))
+                        response = s.get(url=captchasolutionURL,headers=headers,verify=False,proxies=proxies)
 
-                    # Try the categorization request again
-                    response = s.post(url,headers=headers,cookies=c,json=postData,verify=False,proxies=proxies)
+                        # Try the categorization request again
+                        response = s.post(url,headers=headers,json=postData,verify=False,proxies=proxies)
 
-                    responseJSON = json.loads(response.text)
+                        responseJSON = json.loads(response.text)
 
-                    if 'errorType' in responseJSON:
-                        a = responseJSON['errorType']
+                        if 'errorType' in responseJSON:
+                            a = responseJSON['errorType']
+                        else:
+                            a = responseJSON['categorization'][0]['name']
                     else:
-                        a = responseJSON['categorization'][0]['name']
+                        print('[-] Error: Failed to solve BlueCoat CAPTCHA with OCR! Manually solve at "https://sitereview.bluecoat.com/sitereview.jsp"')
                 else:
-                    print('[-] Error: Failed to solve BlueCoat CAPTCHA with OCR! Manually solve at "https://sitereview.bluecoat.com/sitereview.jsp"')
-            else:
-                print('[-] Error: BlueCoat CAPTCHA received. Try --ocr flag or manually solve a CAPTCHA at "https://sitereview.bluecoat.com/sitereview.jsp"')
-
+                    print('[-] Error: BlueCoat CAPTCHA received. Try --ocr flag or manually solve a CAPTCHA at "https://sitereview.bluecoat.com/sitereview.jsp"')
         return a
 
     except Exception as e:
@@ -129,6 +158,8 @@ def checkBluecoat(domain):
         return "error"
 
 def checkIBMXForce(domain):
+    """IBM XForce Domain Reputation"""
+
     try: 
         url = 'https://exchange.xforce.ibmcloud.com/url/{}'.format(domain)
         headers = {'User-Agent':useragent,
@@ -155,7 +186,7 @@ def checkIBMXForce(domain):
         else:
             categories = ''
             # Parse all dictionary keys and append to single string to get Category names
-            for key in responseJSON["result"]['cats']:
+            for key in responseJSON['result']['cats']:
                 categories += '{0}, '.format(str(key))
 
             a = '{0}(Score: {1})'.format(categories,str(responseJSON['result']['score']))
@@ -167,6 +198,8 @@ def checkIBMXForce(domain):
         return "error"
 
 def checkTalos(domain):
+    """Cisco Talos Domain Reputation"""
+
     url = 'https://www.talosintelligence.com/sb_api/query_lookup?query=%2Fapi%2Fv2%2Fdetails%2Fdomain%2F&query_entry={0}&offset=0&order=ip+asc'.format(domain)
     headers = {'User-Agent':useragent,
                'Referer':url}
@@ -195,6 +228,7 @@ def checkTalos(domain):
         return "error"
 
 def checkMXToolbox(domain):
+    """ Checks the MXToolbox service for Google SafeBrowsing and PhishTank information. Currently broken"""
     url = 'https://mxtoolbox.com/Public/Tools/BrandReputation.aspx'
     headers = {'User-Agent':useragent,
             'Origin':url,
@@ -254,6 +288,8 @@ def checkMXToolbox(domain):
         return "error"
 
 def downloadMalwareDomains(malwaredomainsURL):
+    """Downloads a current list of known malicious domains"""
+
     url = malwaredomainsURL
     response = s.get(url=url,headers=headers,verify=False,proxies=proxies)
     responseText = response.text
@@ -263,6 +299,8 @@ def downloadMalwareDomains(malwaredomainsURL):
         print("[-] Error reaching:{}  Status: {}").format(url, response.status_code)
 
 def checkDomain(domain):
+    """Executes various domain reputation checks included in the project"""
+
     print('[*] Fetching domain reputation for: {}'.format(domain))
 
     if domain in maldomainsList:
@@ -277,8 +315,10 @@ def checkDomain(domain):
     ciscotalos = checkTalos(domain)
     print("[+] {}: {}".format(domain, ciscotalos))
 
-    mxtoolbox = checkMXToolbox(domain)
-    print("[+] {}: {}".format(domain, mxtoolbox))
+    #This service has completely changed, removing for now
+    #mxtoolbox = checkMXToolbox(domain)
+    #print("[+] {}: {}".format(domain, mxtoolbox))
+    mxtoolbox = "-"
 
     umbrella = "not available"
     if len(umbrella_apikey):
@@ -291,8 +331,9 @@ def checkDomain(domain):
     return results
 
 def solveCaptcha(url,session):  
-    # Downloads CAPTCHA image and saves to current directory for OCR with tesseract
-    # Returns CAPTCHA string or False if error occured
+    """Downloads CAPTCHA image and saves to current directory for OCR with tesseract
+    Returns CAPTCHA string or False if error occured
+    """
     
     jpeg = 'captcha.jpg'
     
@@ -324,7 +365,7 @@ def solveCaptcha(url,session):
         return False
 
 def drawTable(header,data):
-    
+    """Generates a text based table for printing to the console"""
     data.insert(0,header)
     t = Texttable(max_width=maxwidth)
     t.add_rows(data)
@@ -332,7 +373,9 @@ def drawTable(header,data):
     
     return(t.draw())
 
-def loginExpiredDomains():    
+def loginExpiredDomains():
+    """Login to the ExpiredDomains site with supplied credentials"""
+
     data = "login=%s&password=%s&redirect_2_url=/" % (username, password)
     headers["Content-Type"] = "application/x-www-form-urlencoded"
     r = s.post(expireddomainHost + "/login/", headers=headers, data=data, proxies=proxies, verify=False, allow_redirects=False)
@@ -387,6 +430,7 @@ Examples:
     parser.add_argument('-ks','--keyword-start', help='Keyword starts with used to refine search results', required=False, default="", type=str, dest='keyword_start')
     parser.add_argument('-ke','--keyword-end', help='Keyword ends with used to refine search results', required=False, default="", type=str, dest='keyword_end')
     parser.add_argument('-um','--umbrella-apikey', help='API Key for umbrella (paid)', required=False, default="", type=str, dest='umbrella_apikey')
+    parser.add_argument('-q','--quiet', help='Surpress initial ASCII art and header', required=False, default=False, action='store_true', dest='quiet')
     args = parser.parse_args()
 
     # Load dependent modules
@@ -448,15 +492,14 @@ Examples:
 
     umbrella_apikey = args.umbrella_apikey
 
-    malwaredomainsURL = 'http://mirror1.malwaredomains.com/files/justdomains'
-
+    malwaredomainsURL = 'https://gitlab.com/gerowen/old-malware-domains-ad-list/-/raw/master/malwaredomainslist.txt'
     expireddomainsqueryURL = 'https://www.expireddomains.net/domain-name-search'
     expireddomainHost = "https://member.expireddomains.net"
 
     timestamp = time.strftime("%Y%m%d_%H%M%S")
-            
+
     useragent = 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)'
-   
+
     headers = {'User-Agent':useragent}
 
     proxies = {}
@@ -470,7 +513,7 @@ Examples:
         proxy_parts = urlparse(args.proxy)
         proxies["http"] = "http://%s" % (proxy_parts.netloc)
         proxies["https"] = "https://%s" % (proxy_parts.netloc)
-
+    s.proxies = proxies
     title = '''
  ____   ___  __  __    _    ___ _   _   _   _ _   _ _   _ _____ _____ ____  
 |  _ \ / _ \|  \/  |  / \  |_ _| \ | | | | | | | | | \ | |_   _| ____|  _ \ 
@@ -478,19 +521,19 @@ Examples:
 | |_| | |_| | |  | |/ ___ \ | || |\  | |  _  | |_| | |\  | | | | |___|  _ < 
 |____/ \___/|_|  |_/_/   \_\___|_| \_| |_| |_|\___/|_| \_| |_| |_____|_| \_\ '''
 
-    print(title)
-    print("")
-    print("Expired Domains Reputation Checker")
-    print("Authors: @joevest and @andrewchiles\n")
-    print("DISCLAIMER: This is for educational purposes only!")
-    disclaimer = '''It is designed to promote education and the improvement of computer/cyber security.  
+    # Print header
+    if not (args.quiet):
+        print(title)
+        print('''\nExpired Domains Reputation Checker
+Authors: @joevest and @andrewchiles\n
+DISCLAIMER: This is for educational purposes only!
+It is designed to promote education and the improvement of computer/cyber security.  
 The authors or employers are not liable for any illegal act or misuse performed by any user of this tool.
-If you plan to use this content for illegal purpose, don't.  Have a nice day :)'''
-    print(disclaimer)
-    print("")
+If you plan to use this content for illegal purpose, don't.  Have a nice day :)\n''')
 
     # Download known malware domains
-    print('[*] Downloading malware domain list from {}\n'.format(malwaredomainsURL))
+    # print('[*] Downloading malware domain list from {}\n'.format(malwaredomainsURL))
+    
     maldomains = downloadMalwareDomains(malwaredomainsURL)
     maldomainsList = maldomains.split("\n")
 
@@ -527,7 +570,9 @@ If you plan to use this content for illegal purpose, don't.  Have a nice day :)'
 
     # Generate list of URLs to query for expired/deleted domains
     urls = []
-    
+    if username == None or username == "":
+        print('[-] Error: ExpiredDomains.net requires a username! Use the --username parameter')
+        exit(1)
     if args.password == None or args.password == "":
         password = getpass.getpass("expireddomains.net Password: ")
 

requirements.txt

@@ -4,5 +4,4 @@ beautifulsoup4==4.5.3
 lxml
 pillow
 pytesseract
-urllib3
-uuid
+urllib3