domainhunter.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654
  1. #!/usr/bin/env python
  2. ## Title: domainhunter.py
  3. ## Author: @joevest and @andrewchiles
  4. ## Description: Checks expired domains, reputation/categorization, and Archive.org history to determine
  5. ## good candidates for phishing and C2 domain names
  6. import time
  7. import random
  8. import argparse
  9. import json
  10. import base64
  11. import os
  12. __version__ = "20180506"
  13. ## Functions
  14. def doSleep(timing):
  15. if timing == 0:
  16. time.sleep(random.randrange(90,120))
  17. elif timing == 1:
  18. time.sleep(random.randrange(60,90))
  19. elif timing == 2:
  20. time.sleep(random.randrange(30,60))
  21. elif timing == 3:
  22. time.sleep(random.randrange(10,20))
  23. elif timing == 4:
  24. time.sleep(random.randrange(5,10))
  25. # There's no elif timing == 5 here because we don't want to sleep for -t 5
  26. def checkBluecoat(domain):
  27. try:
  28. url = 'https://sitereview.bluecoat.com/resource/lookup'
  29. postData = {'url':domain,'captcha':''}
  30. headers = {'User-Agent':useragent,
  31. 'Content-Type':'application/json; charset=UTF-8',
  32. 'Referer':'https://sitereview.bluecoat.com/lookup'}
  33. print('[*] BlueCoat: {}'.format(domain))
  34. response = s.post(url,headers=headers,json=postData,verify=False)
  35. responseJSON = json.loads(response.text)
  36. if 'errorType' in responseJSON:
  37. a = responseJSON['errorType']
  38. else:
  39. a = responseJSON['categorization'][0]['name']
  40. # Print notice if CAPTCHAs are blocking accurate results and attempt to solve if --ocr
  41. if a == 'captcha':
  42. if ocr:
  43. # This request is also performed by a browser, but is not needed for our purposes
  44. #captcharequestURL = 'https://sitereview.bluecoat.com/resource/captcha-request'
  45. print('[*] Received CAPTCHA challenge!')
  46. captcha = solveCaptcha('https://sitereview.bluecoat.com/resource/captcha.jpg',s)
  47. if captcha:
  48. b64captcha = base64.urlsafe_b64encode(captcha.encode('utf-8')).decode('utf-8')
  49. # Send CAPTCHA solution via GET since inclusion with the domain categorization request doens't work anymore
  50. captchasolutionURL = 'https://sitereview.bluecoat.com/resource/captcha-request/{0}'.format(b64captcha)
  51. print('[*] Submiting CAPTCHA at {0}'.format(captchasolutionURL))
  52. response = s.get(url=captchasolutionURL,headers=headers,verify=False)
  53. # Try the categorization request again
  54. response = s.post(url,headers=headers,json=postData,verify=False)
  55. responseJSON = json.loads(response.text)
  56. if 'errorType' in responseJSON:
  57. a = responseJSON['errorType']
  58. else:
  59. a = responseJSON['categorization'][0]['name']
  60. else:
  61. print('[-] Error: Failed to solve BlueCoat CAPTCHA with OCR! Manually solve at "https://sitereview.bluecoat.com/sitereview.jsp"')
  62. else:
  63. print('[-] Error: BlueCoat CAPTCHA received. Try --ocr flag or manually solve a CAPTCHA at "https://sitereview.bluecoat.com/sitereview.jsp"')
  64. return a
  65. except Exception as e:
  66. print('[-] Error retrieving Bluecoat reputation! {0}'.format(e))
  67. return "-"
  68. def checkIBMXForce(domain):
  69. try:
  70. url = 'https://exchange.xforce.ibmcloud.com/url/{}'.format(domain)
  71. headers = {'User-Agent':useragent,
  72. 'Accept':'application/json, text/plain, */*',
  73. 'x-ui':'XFE',
  74. 'Origin':url,
  75. 'Referer':url}
  76. print('[*] IBM xForce: {}'.format(domain))
  77. url = 'https://api.xforce.ibmcloud.com/url/{}'.format(domain)
  78. response = s.get(url,headers=headers,verify=False)
  79. responseJSON = json.loads(response.text)
  80. if 'error' in responseJSON:
  81. a = responseJSON['error']
  82. elif not responseJSON['result']['cats']:
  83. a = 'Uncategorized'
  84. ## TO-DO - Add noticed when "intrusion" category is returned. This is indication of rate limit / brute-force protection hit on the endpoint
  85. else:
  86. categories = ''
  87. # Parse all dictionary keys and append to single string to get Category names
  88. for key in responseJSON["result"]['cats']:
  89. categories += '{0}, '.format(str(key))
  90. a = '{0}(Score: {1})'.format(categories,str(responseJSON['result']['score']))
  91. return a
  92. except:
  93. print('[-] Error retrieving IBM x-Force reputation!')
  94. return "-"
  95. def checkTalos(domain):
  96. url = 'https://www.talosintelligence.com/sb_api/query_lookup?query=%2Fapi%2Fv2%2Fdetails%2Fdomain%2F&query_entry={0}&offset=0&order=ip+asc'.format(domain)
  97. headers = {'User-Agent':useragent,
  98. 'Referer':url}
  99. print('[*] Cisco Talos: {}'.format(domain))
  100. try:
  101. response = s.get(url,headers=headers,verify=False)
  102. responseJSON = json.loads(response.text)
  103. if 'error' in responseJSON:
  104. a = str(responseJSON['error'])
  105. if a == "Unfortunately, we can't find any results for your search.":
  106. a = 'Uncategorized'
  107. elif responseJSON['category'] is None:
  108. a = 'Uncategorized'
  109. else:
  110. a = '{0} (Score: {1})'.format(str(responseJSON['category']['description']), str(responseJSON['web_score_name']))
  111. return a
  112. except:
  113. print('[-] Error retrieving Talos reputation!')
  114. return "-"
  115. def checkMXToolbox(domain):
  116. url = 'https://mxtoolbox.com/Public/Tools/BrandReputation.aspx'
  117. headers = {'User-Agent':useragent,
  118. 'Origin':url,
  119. 'Referer':url}
  120. print('[*] Google SafeBrowsing and PhishTank: {}'.format(domain))
  121. try:
  122. response = s.get(url=url, headers=headers)
  123. soup = BeautifulSoup(response.content,'lxml')
  124. viewstate = soup.select('input[name=__VIEWSTATE]')[0]['value']
  125. viewstategenerator = soup.select('input[name=__VIEWSTATEGENERATOR]')[0]['value']
  126. eventvalidation = soup.select('input[name=__EVENTVALIDATION]')[0]['value']
  127. data = {
  128. "__EVENTTARGET": "",
  129. "__EVENTARGUMENT": "",
  130. "__VIEWSTATE": viewstate,
  131. "__VIEWSTATEGENERATOR": viewstategenerator,
  132. "__EVENTVALIDATION": eventvalidation,
  133. "ctl00$ContentPlaceHolder1$brandReputationUrl": domain,
  134. "ctl00$ContentPlaceHolder1$brandReputationDoLookup": "Brand Reputation Lookup",
  135. "ctl00$ucSignIn$hfRegCode": 'missing',
  136. "ctl00$ucSignIn$hfRedirectSignUp": '/Public/Tools/BrandReputation.aspx',
  137. "ctl00$ucSignIn$hfRedirectLogin": '',
  138. "ctl00$ucSignIn$txtEmailAddress": '',
  139. "ctl00$ucSignIn$cbNewAccount": 'cbNewAccount',
  140. "ctl00$ucSignIn$txtFullName": '',
  141. "ctl00$ucSignIn$txtModalNewPassword": '',
  142. "ctl00$ucSignIn$txtPhone": '',
  143. "ctl00$ucSignIn$txtCompanyName": '',
  144. "ctl00$ucSignIn$drpTitle": '',
  145. "ctl00$ucSignIn$txtTitleName": '',
  146. "ctl00$ucSignIn$txtModalPassword": ''
  147. }
  148. response = s.post(url=url, headers=headers, data=data)
  149. soup = BeautifulSoup(response.content,'lxml')
  150. a = ''
  151. if soup.select('div[id=ctl00_ContentPlaceHolder1_noIssuesFound]'):
  152. a = 'No issues found'
  153. return a
  154. else:
  155. if soup.select('div[id=ctl00_ContentPlaceHolder1_googleSafeBrowsingIssuesFound]'):
  156. a = 'Google SafeBrowsing Issues Found. '
  157. if soup.select('div[id=ctl00_ContentPlaceHolder1_phishTankIssuesFound]'):
  158. a += 'PhishTank Issues Found'
  159. return a
  160. except Exception as e:
  161. print('[-] Error retrieving Google SafeBrowsing and PhishTank reputation!')
  162. return "-"
  163. def downloadMalwareDomains(malwaredomainsURL):
  164. url = malwaredomainsURL
  165. response = s.get(url=url,headers=headers,verify=False)
  166. responseText = response.text
  167. if response.status_code == 200:
  168. return responseText
  169. else:
  170. print("[-] Error reaching:{} Status: {}").format(url, response.status_code)
  171. def checkDomain(domain):
  172. print('[*] Fetching domain reputation for: {}'.format(domain))
  173. if domain in maldomainsList:
  174. print("[!] {}: Identified as known malware domain (malwaredomains.com)".format(domain))
  175. bluecoat = checkBluecoat(domain)
  176. print("[+] {}: {}".format(domain, bluecoat))
  177. ibmxforce = checkIBMXForce(domain)
  178. print("[+] {}: {}".format(domain, ibmxforce))
  179. ciscotalos = checkTalos(domain)
  180. print("[+] {}: {}".format(domain, ciscotalos))
  181. mxtoolbox = checkMXToolbox(domain)
  182. print("[+] {}: {}".format(domain, mxtoolbox))
  183. print("")
  184. results = [domain,bluecoat,ibmxforce,ciscotalos,mxtoolbox]
  185. return results
  186. def solveCaptcha(url,session):
  187. # Downloads CAPTCHA image and saves to current directory for OCR with tesseract
  188. # Returns CAPTCHA string or False if error occured
  189. jpeg = 'captcha.jpg'
  190. try:
  191. response = session.get(url=url,headers=headers,verify=False, stream=True)
  192. if response.status_code == 200:
  193. with open(jpeg, 'wb') as f:
  194. response.raw.decode_content = True
  195. shutil.copyfileobj(response.raw, f)
  196. else:
  197. print('[-] Error downloading CAPTCHA file!')
  198. return False
  199. # Perform basic OCR without additional image enhancement
  200. text = pytesseract.image_to_string(Image.open(jpeg))
  201. text = text.replace(" ", "")
  202. # Remove CAPTCHA file
  203. try:
  204. os.remove(jpeg)
  205. except OSError:
  206. pass
  207. return text
  208. except Exception as e:
  209. print("[-] Error solving CAPTCHA - {0}".format(e))
  210. return False
  211. def drawTable(header,data):
  212. data.insert(0,header)
  213. t = Texttable(max_width=maxwidth)
  214. t.add_rows(data)
  215. t.header(header)
  216. return(t.draw())
  217. ## MAIN
  218. if __name__ == "__main__":
  219. parser = argparse.ArgumentParser(description='Finds expired domains, domain categorization, and Archive.org history to determine good candidates for C2 and phishing domains')
  220. parser.add_argument('-k','--keyword', help='Keyword used to refine search results', required=False, default=False, type=str, dest='keyword')
  221. parser.add_argument('-c','--check', help='Perform domain reputation checks', required=False, default=False, action='store_true', dest='check')
  222. parser.add_argument('-f','--filename', help='Specify input file of line delimited domain names to check', required=False, default=False, type=str, dest='filename')
  223. parser.add_argument('--ocr', help='Perform OCR on CAPTCHAs when challenged', required=False, default=False, action='store_true')
  224. parser.add_argument('-r','--maxresults', help='Number of results to return when querying latest expired/deleted domains', required=False, default=100, type=int, dest='maxresults')
  225. parser.add_argument('-s','--single', help='Performs detailed reputation checks against a single domain name/IP.', required=False, default=False, dest='single')
  226. parser.add_argument('-t','--timing', help='Modifies request timing to avoid CAPTCHAs. Slowest(0) = 90-120 seconds, Default(3) = 10-20 seconds, Fastest(5) = no delay', required=False, default=3, type=int, choices=range(0,6), dest='timing')
  227. parser.add_argument('-w','--maxwidth', help='Width of text table', required=False, default=400, type=int, dest='maxwidth')
  228. parser.add_argument('-V','--version', action='version',version='%(prog)s {version}'.format(version=__version__))
  229. args = parser.parse_args()
  230. # Load dependent modules
  231. try:
  232. import requests
  233. from bs4 import BeautifulSoup
  234. from texttable import Texttable
  235. except Exception as e:
  236. print("Expired Domains Reputation Check")
  237. print("[-] Missing basic dependencies: {}".format(str(e)))
  238. print("[*] Install required dependencies by running `pip3 install -r requirements.txt`")
  239. quit(0)
  240. # Load OCR related modules if --ocr flag is set since these can be difficult to get working
  241. if args.ocr:
  242. try:
  243. import pytesseract
  244. from PIL import Image
  245. import shutil
  246. except Exception as e:
  247. print("Expired Domains Reputation Check")
  248. print("[-] Missing OCR dependencies: {}".format(str(e)))
  249. print("[*] Install required Python dependencies by running: pip3 install -r requirements.txt")
  250. print("[*] Ubuntu\Debian - Install tesseract by running: apt-get install tesseract-ocr python3-imaging")
  251. print("[*] macOS - Install tesseract with homebrew by running: brew install tesseract")
  252. quit(0)
  253. ## Variables
  254. keyword = args.keyword
  255. check = args.check
  256. filename = args.filename
  257. maxresults = args.maxresults
  258. single = args.single
  259. timing = args.timing
  260. maxwidth = args.maxwidth
  261. ocr = args.ocr
  262. malwaredomainsURL = 'http://mirror1.malwaredomains.com/files/justdomains'
  263. expireddomainsqueryURL = 'https://www.expireddomains.net/domain-name-search'
  264. timestamp = time.strftime("%Y%m%d_%H%M%S")
  265. useragent = 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)'
  266. headers = {'User-Agent':useragent}
  267. requests.packages.urllib3.disable_warnings()
  268. # HTTP Session container, used to manage cookies, session tokens and other session information
  269. s = requests.Session()
  270. title = '''
  271. ____ ___ __ __ _ ___ _ _ _ _ _ _ _ _ _____ _____ ____
  272. | _ \ / _ \| \/ | / \ |_ _| \ | | | | | | | | | \ | |_ _| ____| _ \
  273. | | | | | | | |\/| | / _ \ | || \| | | |_| | | | | \| | | | | _| | |_) |
  274. | |_| | |_| | | | |/ ___ \ | || |\ | | _ | |_| | |\ | | | | |___| _ <
  275. |____/ \___/|_| |_/_/ \_\___|_| \_| |_| |_|\___/|_| \_| |_| |_____|_| \_\ '''
  276. print(title)
  277. print("")
  278. print("Expired Domains Reputation Checker")
  279. print("Authors: @joevest and @andrewchiles\n")
  280. print("DISCLAIMER: This is for educational purposes only!")
  281. disclaimer = '''It is designed to promote education and the improvement of computer/cyber security.
  282. The authors or employers are not liable for any illegal act or misuse performed by any user of this tool.
  283. If you plan to use this content for illegal purpose, don't. Have a nice day :)'''
  284. print(disclaimer)
  285. print("")
  286. # Download known malware domains
  287. print('[*] Downloading malware domain list from {}\n'.format(malwaredomainsURL))
  288. maldomains = downloadMalwareDomains(malwaredomainsURL)
  289. maldomainsList = maldomains.split("\n")
  290. # Retrieve reputation for a single choosen domain (Quick Mode)
  291. if single:
  292. checkDomain(single)
  293. exit(0)
  294. # Perform detailed domain reputation checks against input file, print table, and quit
  295. if filename:
  296. # Initialize our list with an empty row for the header
  297. data = []
  298. try:
  299. with open(filename, 'r') as domainsList:
  300. for line in domainsList.read().splitlines():
  301. data.append(checkDomain(line))
  302. doSleep(timing)
  303. # Print results table
  304. header = ['Domain', 'BlueCoat', 'IBM X-Force', 'Cisco Talos', 'MXToolbox']
  305. print(drawTable(header,data))
  306. except KeyboardInterrupt:
  307. print('Caught keyboard interrupt. Exiting!')
  308. exit(0)
  309. except Exception as e:
  310. print('[-] Error: {}'.format(e))
  311. exit(1)
  312. exit(0)
  313. # Generic Proxy support
  314. # TODO: add as a parameter
  315. proxies = {
  316. 'http': 'http://127.0.0.1:8080',
  317. 'https': 'http://127.0.0.1:8080',
  318. }
  319. # Create an initial session
  320. domainrequest = s.get("https://www.expireddomains.net",headers=headers,verify=False)
  321. # Use proxy like Burp for debugging request/parsing errors
  322. #domainrequest = s.get("https://www.expireddomains.net",headers=headers,verify=False,proxies=proxies)
  323. # Lists for our ExpiredDomains results
  324. domain_list = []
  325. data = []
  326. # Generate list of URLs to query for expired/deleted domains
  327. urls = []
  328. # Use the keyword string to narrow domain search if provided. This generates a list of URLs to query
  329. if keyword:
  330. print('[*] Fetching expired or deleted domains containing "{}"'.format(keyword))
  331. for i in range (0,maxresults,25):
  332. if i == 0:
  333. urls.append("{}/?q={}&fwhois=22&falexa=1".format(expireddomainsqueryURL,keyword))
  334. headers['Referer'] ='https://www.expireddomains.net/domain-name-search/?q={}&start=1'.format(keyword)
  335. else:
  336. urls.append("{}/?start={}&q={}&fwhois=22&falexa=1".format(expireddomainsqueryURL,i,keyword))
  337. headers['Referer'] ='https://www.expireddomains.net/domain-name-search/?start={}&q={}'.format((i-25),keyword)
  338. # If no keyword provided, generate list of recently expired domains URLS (batches of 25 results).
  339. else:
  340. print('[*] Fetching expired or deleted domains...')
  341. # Caculate number of URLs to request since we're performing a request for two different resources instead of one
  342. numresults = int(maxresults / 2)
  343. for i in range (0,(numresults),25):
  344. urls.append('https://www.expireddomains.net/backorder-expired-domains?start={}&ftlds[]=2&ftlds[]=3&ftlds[]=4&falexa=1'.format(i))
  345. urls.append('https://www.expireddomains.net/deleted-com-domains/?start={}&ftlds[]=2&ftlds[]=3&ftlds[]=4&falexa=1'.format(i))
  346. for url in urls:
  347. print("[*] {}".format(url))
  348. # Annoyingly when querying specific keywords the expireddomains.net site requires additional cookies which
  349. # are set in JavaScript and not recognized by Requests so we add them here manually.
  350. # May not be needed, but the _pk_id.10.dd0a cookie only requires a single . to be successful
  351. # In order to somewhat match a real cookie, but still be different, random integers are introduced
  352. r1 = random.randint(100000,999999)
  353. # Known good example _pk_id.10.dd0a cookie: 5abbbc772cbacfb1.1496760705.2.1496760705.1496760705
  354. pk_str = '5abbbc772cbacfb1' + '.1496' + str(r1) + '.2.1496' + str(r1) + '.1496' + str(r1)
  355. jar = requests.cookies.RequestsCookieJar()
  356. jar.set('_pk_ses.10.dd0a', '*', domain='expireddomains.net', path='/')
  357. jar.set('_pk_id.10.dd0a', pk_str, domain='expireddomains.net', path='/')
  358. domainrequest = s.get(url,headers=headers,verify=False,cookies=jar)
  359. #domainrequest = s.get(url,headers=headers,verify=False,cookies=jar,proxies=proxies)
  360. domains = domainrequest.text
  361. # Turn the HTML into a Beautiful Soup object
  362. soup = BeautifulSoup(domains, 'lxml')
  363. #print(soup)
  364. try:
  365. table = soup.find("table")
  366. for row in table.findAll('tr')[1:]:
  367. # Alternative way to extract domain name
  368. # domain = row.find('td').find('a').text
  369. cells = row.findAll("td")
  370. if len(cells) >= 1:
  371. if keyword:
  372. c0 = row.find('td').find('a').text # domain
  373. c1 = cells[1].find(text=True) # bl
  374. c2 = cells[2].find(text=True) # domainpop
  375. c3 = cells[3].find(text=True) # birth
  376. c4 = cells[4].find(text=True) # Archive.org entries
  377. c5 = cells[5].find(text=True) # similarweb
  378. c6 = cells[6].find(text=True) # similarweb country code
  379. c7 = cells[7].find(text=True) # Dmoz.org
  380. c8 = cells[8].find(text=True) # status com
  381. c9 = cells[9].find(text=True) # status net
  382. c10 = cells[10].find(text=True) # status org
  383. c11 = cells[11].find(text=True) # status de
  384. c12 = cells[12].find(text=True) # tld registered
  385. c13 = cells[13].find(text=True) # Source List
  386. c14 = cells[14].find(text=True) # Domain Status
  387. c15 = "" # Related Domains
  388. # Non-keyword search table format is slightly different
  389. else:
  390. c0 = cells[0].find(text=True) # domain
  391. c1 = cells[1].find(text=True) # bl
  392. c2 = cells[2].find(text=True) # domainpop
  393. c3 = cells[3].find(text=True) # birth
  394. c4 = cells[4].find(text=True) # Archive.org entries
  395. c5 = cells[5].find(text=True) # similarweb
  396. c6 = cells[6].find(text=True) # similarweb country code
  397. c7 = cells[7].find(text=True) # Dmoz.org
  398. c8 = cells[8].find(text=True) # status com
  399. c9 = cells[9].find(text=True) # status net
  400. c10 = cells[10].find(text=True) # status org
  401. c11 = cells[11].find(text=True) # status de
  402. c12 = cells[12].find(text=True) # tld registered
  403. c13 = cells[13].find(text=True) # changes
  404. c14 = cells[14].find(text=True) # whois
  405. available = ''
  406. if c8 == "available":
  407. available += ".com "
  408. if c9 == "available":
  409. available += ".net "
  410. if c10 == "available":
  411. available += ".org "
  412. if c11 == "available":
  413. available += ".de "
  414. # Only grab status for keyword searches since it doesn't exist otherwise
  415. status = ""
  416. if keyword:
  417. status = c14
  418. # Append parsed domain data to list if it matches our criteria (.com|.net|.org and not a known malware domain)
  419. if (c0.lower().endswith(".com") or c0.lower().endswith(".net") or c0.lower().endswith(".org")) and (c0 not in maldomainsList):
  420. domain_list.append([c0,c3,c4,available,status])
  421. except Exception as e:
  422. #print(e)
  423. pass
  424. # Add additional sleep on requests to ExpiredDomains.net to avoid errors
  425. time.sleep(5)
  426. # Check for valid list results before continuing
  427. if len(domain_list) == 0:
  428. print("[-] No domain results found or none are currently available for purchase!")
  429. exit(0)
  430. else:
  431. if check:
  432. print("\n[*] Performing reputation checks for {} domains".format(len(domain_list)))
  433. for domain_entry in domain_list:
  434. domain = domain_entry[0]
  435. birthdate = domain_entry[1]
  436. archiveentries = domain_entry[2]
  437. availabletlds = domain_entry[3]
  438. status = domain_entry[4]
  439. bluecoat = ''
  440. ibmxforce = ''
  441. ciscotalos = ''
  442. # Perform domain reputation checks
  443. if check:
  444. bluecoat = checkBluecoat(domain)
  445. print("[+] {}: {}".format(domain, bluecoat))
  446. ibmxforce = checkIBMXForce(domain)
  447. print("[+] {}: {}".format(domain, ibmxforce))
  448. ciscotalos = checkTalos(domain)
  449. print("[+] {}: {}".format(domain, ciscotalos))
  450. # Sleep to avoid captchas
  451. doSleep(timing)
  452. # Mark reputation checks as skipped if -c flag not present
  453. else:
  454. bluecoat = '-'
  455. ibmxforce = '-'
  456. ciscotalos = '-'
  457. # Append entry to new list with reputation if at least one service reports reputation
  458. if not ((bluecoat in ('Uncategorized','badurl','Suspicious','Malicious Sources/Malnets','captcha','phishing')) and ibmxforce == "Not found." and ciscotalos == "Uncategorized"):
  459. data.append([domain,birthdate,archiveentries,availabletlds,status,bluecoat,ibmxforce,ciscotalos])
  460. # Sort domain list by column 2 (Birth Year)
  461. sortedDomains = sorted(data, key=lambda x: x[1], reverse=True)
  462. if len(sortedDomains) == 0:
  463. print("\n[-] No domains discovered with a desireable categorization!")
  464. exit(0)
  465. else:
  466. print("\n[*] {} of {} domains discovered with a potentially desireable categorization!".format(len(sortedDomains),len(domain_list)))
  467. # Build HTML Table
  468. html = ''
  469. htmlHeader = '<html><head><title>Expired Domain List</title></head>'
  470. htmlBody = '<body><p>The following available domains report was generated at {}</p>'.format(timestamp)
  471. htmlTableHeader = '''
  472. <table border="1" align="center">
  473. <th>Domain</th>
  474. <th>Birth</th>
  475. <th>Entries</th>
  476. <th>TLDs Available</th>
  477. <th>Status</th>
  478. <th>BlueCoat</th>
  479. <th>IBM X-Force</th>
  480. <th>Cisco Talos</th>
  481. <th>WatchGuard</th>
  482. <th>Namecheap</th>
  483. <th>Archive.org</th>
  484. '''
  485. htmlTableBody = ''
  486. htmlTableFooter = '</table>'
  487. htmlFooter = '</body></html>'
  488. # Build HTML table contents
  489. for i in sortedDomains:
  490. htmlTableBody += '<tr>'
  491. htmlTableBody += '<td>{}</td>'.format(i[0]) # Domain
  492. htmlTableBody += '<td>{}</td>'.format(i[1]) # Birth
  493. htmlTableBody += '<td>{}</td>'.format(i[2]) # Entries
  494. htmlTableBody += '<td>{}</td>'.format(i[3]) # TLDs
  495. htmlTableBody += '<td>{}</td>'.format(i[4]) # Status
  496. htmlTableBody += '<td><a href="https://sitereview.bluecoat.com/" target="_blank">{}</a></td>'.format(i[5]) # Bluecoat
  497. htmlTableBody += '<td><a href="https://exchange.xforce.ibmcloud.com/url/{}" target="_blank">{}</a></td>'.format(i[0],i[6]) # IBM x-Force Categorization
  498. htmlTableBody += '<td><a href="https://www.talosintelligence.com/reputation_center/lookup?search={}" target="_blank">{}</a></td>'.format(i[0],i[7]) # Cisco Talos
  499. htmlTableBody += '<td><a href="http://www.borderware.com/domain_lookup.php?ip={}" target="_blank">WatchGuard</a></td>'.format(i[0]) # Borderware WatchGuard
  500. htmlTableBody += '<td><a href="https://www.namecheap.com/domains/registration/results.aspx?domain={}" target="_blank">Namecheap</a></td>'.format(i[0]) # Namecheap
  501. htmlTableBody += '<td><a href="http://web.archive.org/web/*/{}" target="_blank">Archive.org</a></td>'.format(i[0]) # Archive.org
  502. htmlTableBody += '</tr>'
  503. html = htmlHeader + htmlBody + htmlTableHeader + htmlTableBody + htmlTableFooter + htmlFooter
  504. logfilename = "{}_domainreport.html".format(timestamp)
  505. log = open(logfilename,'w')
  506. log.write(html)
  507. log.close
  508. print("\n[*] Search complete")
  509. print("[*] Log written to {}\n".format(logfilename))
  510. # Print Text Table
  511. header = ['Domain', 'Birth', '#', 'TLDs', 'Status', 'BlueCoat', 'IBM', 'Cisco Talos']
  512. print(drawTable(header,sortedDomains))