domainhunter.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715
  1. #!/usr/bin/env python
  2. ## Title: domainhunter.py
  3. ## Author: @joevest and @andrewchiles
  4. ## Description: Checks expired domains, reputation/categorization, and Archive.org history to determine
  5. ## good candidates for phishing and C2 domain names
  6. # If the expected response format from a provider changes, use the traceback module to get a full stack trace without removing try/catch blocks
  7. #import traceback
  8. #traceback.print_exc()
  9. import time
  10. import random
  11. import argparse
  12. import json
  13. import base64
  14. import os
  15. import sys
  16. from urllib.parse import urlparse
  17. import getpass
  18. import uuid
  19. __version__ = "20190716"
  20. ## Functions
  21. def doSleep(timing):
  22. if timing == 0:
  23. time.sleep(random.randrange(90,120))
  24. elif timing == 1:
  25. time.sleep(random.randrange(60,90))
  26. elif timing == 2:
  27. time.sleep(random.randrange(30,60))
  28. elif timing == 3:
  29. time.sleep(random.randrange(10,20))
  30. elif timing == 4:
  31. time.sleep(random.randrange(5,10))
  32. # There's no elif timing == 5 here because we don't want to sleep for -t 5
  33. def checkBluecoat(domain):
  34. try:
  35. url = 'https://sitereview.bluecoat.com/resource/lookup'
  36. postData = {'url':domain,'captcha':''}
  37. token = str(uuid.uuid4())
  38. headers = {'User-Agent':useragent,
  39. 'Accept':'application/json, text/plain, */*',
  40. 'Accept-Language':'en_US',
  41. 'Content-Type':'application/json; charset=UTF-8',
  42. 'X-XSRF-TOKEN':token,
  43. 'Referer':'http://sitereview.bluecoat.com/'}
  44. c = {
  45. "JSESSIONID":str(uuid.uuid4()).upper().replace("-", ""),
  46. "XSRF-TOKEN":token
  47. }
  48. print('[*] BlueCoat: {}'.format(domain))
  49. response = s.post(url,headers=headers,cookies=c,json=postData,verify=False,proxies=proxies)
  50. responseJSON = json.loads(response.text)
  51. if 'errorType' in responseJSON:
  52. a = responseJSON['errorType']
  53. else:
  54. a = responseJSON['categorization'][0]['name']
  55. # Print notice if CAPTCHAs are blocking accurate results and attempt to solve if --ocr
  56. if a == 'captcha':
  57. if ocr:
  58. # This request is also performed by a browser, but is not needed for our purposes
  59. #captcharequestURL = 'https://sitereview.bluecoat.com/resource/captcha-request'
  60. print('[*] Received CAPTCHA challenge!')
  61. captcha = solveCaptcha('https://sitereview.bluecoat.com/resource/captcha.jpg',s)
  62. if captcha:
  63. b64captcha = base64.urlsafe_b64encode(captcha.encode('utf-8')).decode('utf-8')
  64. # Send CAPTCHA solution via GET since inclusion with the domain categorization request doens't work anymore
  65. captchasolutionURL = 'https://sitereview.bluecoat.com/resource/captcha-request/{0}'.format(b64captcha)
  66. print('[*] Submiting CAPTCHA at {0}'.format(captchasolutionURL))
  67. response = s.get(url=captchasolutionURL,headers=headers,verify=False,proxies=proxies)
  68. # Try the categorization request again
  69. response = s.post(url,headers=headers,json=postData,verify=False,proxies=proxies)
  70. responseJSON = json.loads(response.text)
  71. if 'errorType' in responseJSON:
  72. a = responseJSON['errorType']
  73. else:
  74. a = responseJSON['categorization'][0]['name']
  75. else:
  76. print('[-] Error: Failed to solve BlueCoat CAPTCHA with OCR! Manually solve at "https://sitereview.bluecoat.com/sitereview.jsp"')
  77. else:
  78. print('[-] Error: BlueCoat CAPTCHA received. Try --ocr flag or manually solve a CAPTCHA at "https://sitereview.bluecoat.com/sitereview.jsp"')
  79. return a
  80. except Exception as e:
  81. print('[-] Error retrieving Bluecoat reputation! {0}'.format(e))
  82. return "error"
  83. def checkIBMXForce(domain):
  84. try:
  85. url = 'https://exchange.xforce.ibmcloud.com/url/{}'.format(domain)
  86. headers = {'User-Agent':useragent,
  87. 'Accept':'application/json, text/plain, */*',
  88. 'x-ui':'XFE',
  89. 'Origin':url,
  90. 'Referer':url}
  91. print('[*] IBM xForce: {}'.format(domain))
  92. url = 'https://api.xforce.ibmcloud.com/url/{}'.format(domain)
  93. response = s.get(url,headers=headers,verify=False,proxies=proxies)
  94. responseJSON = json.loads(response.text)
  95. if 'error' in responseJSON:
  96. a = responseJSON['error']
  97. elif not responseJSON['result']['cats']:
  98. a = 'Uncategorized'
  99. ## TO-DO - Add noticed when "intrusion" category is returned. This is indication of rate limit / brute-force protection hit on the endpoint
  100. else:
  101. categories = ''
  102. # Parse all dictionary keys and append to single string to get Category names
  103. for key in responseJSON["result"]['cats']:
  104. categories += '{0}, '.format(str(key))
  105. a = '{0}(Score: {1})'.format(categories,str(responseJSON['result']['score']))
  106. return a
  107. except Exception as e:
  108. print('[-] Error retrieving IBM-Xforce reputation! {0}'.format(e))
  109. return "error"
  110. def checkTalos(domain):
  111. url = 'https://www.talosintelligence.com/sb_api/query_lookup?query=%2Fapi%2Fv2%2Fdetails%2Fdomain%2F&query_entry={0}&offset=0&order=ip+asc'.format(domain)
  112. headers = {'User-Agent':useragent,
  113. 'Referer':url}
  114. print('[*] Cisco Talos: {}'.format(domain))
  115. try:
  116. response = s.get(url,headers=headers,verify=False,proxies=proxies)
  117. responseJSON = json.loads(response.text)
  118. if 'error' in responseJSON:
  119. a = str(responseJSON['error'])
  120. if a == "Unfortunately, we can't find any results for your search.":
  121. a = 'Uncategorized'
  122. elif responseJSON['category'] is None:
  123. a = 'Uncategorized'
  124. else:
  125. a = '{0} (Score: {1})'.format(str(responseJSON['category']['description']), str(responseJSON['web_score_name']))
  126. return a
  127. except Exception as e:
  128. print('[-] Error retrieving Talos reputation! {0}'.format(e))
  129. return "error"
  130. def checkMXToolbox(domain):
  131. url = 'https://mxtoolbox.com/Public/Tools/BrandReputation.aspx'
  132. headers = {'User-Agent':useragent,
  133. 'Origin':url,
  134. 'Referer':url}
  135. print('[*] Google SafeBrowsing and PhishTank: {}'.format(domain))
  136. try:
  137. response = s.get(url=url, headers=headers,proxies=proxies,verify=False)
  138. soup = BeautifulSoup(response.content,'lxml')
  139. viewstate = soup.select('input[name=__VIEWSTATE]')[0]['value']
  140. viewstategenerator = soup.select('input[name=__VIEWSTATEGENERATOR]')[0]['value']
  141. eventvalidation = soup.select('input[name=__EVENTVALIDATION]')[0]['value']
  142. data = {
  143. "__EVENTTARGET": "",
  144. "__EVENTARGUMENT": "",
  145. "__VIEWSTATE": viewstate,
  146. "__VIEWSTATEGENERATOR": viewstategenerator,
  147. "__EVENTVALIDATION": eventvalidation,
  148. "ctl00$ContentPlaceHolder1$brandReputationUrl": domain,
  149. "ctl00$ContentPlaceHolder1$brandReputationDoLookup": "Brand Reputation Lookup",
  150. "ctl00$ucSignIn$hfRegCode": 'missing',
  151. "ctl00$ucSignIn$hfRedirectSignUp": '/Public/Tools/BrandReputation.aspx',
  152. "ctl00$ucSignIn$hfRedirectLogin": '',
  153. "ctl00$ucSignIn$txtEmailAddress": '',
  154. "ctl00$ucSignIn$cbNewAccount": 'cbNewAccount',
  155. "ctl00$ucSignIn$txtFullName": '',
  156. "ctl00$ucSignIn$txtModalNewPassword": '',
  157. "ctl00$ucSignIn$txtPhone": '',
  158. "ctl00$ucSignIn$txtCompanyName": '',
  159. "ctl00$ucSignIn$drpTitle": '',
  160. "ctl00$ucSignIn$txtTitleName": '',
  161. "ctl00$ucSignIn$txtModalPassword": ''
  162. }
  163. response = s.post(url=url, headers=headers, data=data,proxies=proxies,verify=False)
  164. soup = BeautifulSoup(response.content,'lxml')
  165. a = ''
  166. if soup.select('div[id=ctl00_ContentPlaceHolder1_noIssuesFound]'):
  167. a = 'No issues found'
  168. return a
  169. else:
  170. if soup.select('div[id=ctl00_ContentPlaceHolder1_googleSafeBrowsingIssuesFound]'):
  171. a = 'Google SafeBrowsing Issues Found. '
  172. if soup.select('div[id=ctl00_ContentPlaceHolder1_phishTankIssuesFound]'):
  173. a += 'PhishTank Issues Found'
  174. return a
  175. except Exception as e:
  176. print('[-] Error retrieving Google SafeBrowsing and PhishTank reputation!')
  177. return "error"
  178. def downloadMalwareDomains(malwaredomainsURL):
  179. url = malwaredomainsURL
  180. response = s.get(url=url,headers=headers,verify=False,proxies=proxies)
  181. responseText = response.text
  182. if response.status_code == 200:
  183. return responseText
  184. else:
  185. print("[-] Error reaching:{} Status: {}").format(url, response.status_code)
  186. def checkDomain(domain):
  187. print('[*] Fetching domain reputation for: {}'.format(domain))
  188. if domain in maldomainsList:
  189. print("[!] {}: Identified as known malware domain (malwaredomains.com)".format(domain))
  190. bluecoat = checkBluecoat(domain)
  191. print("[+] {}: {}".format(domain, bluecoat))
  192. ibmxforce = checkIBMXForce(domain)
  193. print("[+] {}: {}".format(domain, ibmxforce))
  194. ciscotalos = checkTalos(domain)
  195. print("[+] {}: {}".format(domain, ciscotalos))
  196. mxtoolbox = checkMXToolbox(domain)
  197. print("[+] {}: {}".format(domain, mxtoolbox))
  198. print("")
  199. results = [domain,bluecoat,ibmxforce,ciscotalos,mxtoolbox]
  200. return results
  201. def solveCaptcha(url,session):
  202. # Downloads CAPTCHA image and saves to current directory for OCR with tesseract
  203. # Returns CAPTCHA string or False if error occured
  204. jpeg = 'captcha.jpg'
  205. try:
  206. response = session.get(url=url,headers=headers,verify=False, stream=True,proxies=proxies)
  207. if response.status_code == 200:
  208. with open(jpeg, 'wb') as f:
  209. response.raw.decode_content = True
  210. shutil.copyfileobj(response.raw, f)
  211. else:
  212. print('[-] Error downloading CAPTCHA file!')
  213. return False
  214. # Perform basic OCR without additional image enhancement
  215. text = pytesseract.image_to_string(Image.open(jpeg))
  216. text = text.replace(" ", "")
  217. # Remove CAPTCHA file
  218. try:
  219. os.remove(jpeg)
  220. except OSError:
  221. pass
  222. return text
  223. except Exception as e:
  224. print("[-] Error solving CAPTCHA - {0}".format(e))
  225. return False
  226. def drawTable(header,data):
  227. data.insert(0,header)
  228. t = Texttable(max_width=maxwidth)
  229. t.add_rows(data)
  230. t.header(header)
  231. return(t.draw())
  232. def loginExpiredDomains():
  233. data = "login=%s&password=%s&redirect_2_url=/" % (username, password)
  234. headers["Content-Type"] = "application/x-www-form-urlencoded"
  235. r = s.post(expireddomainHost + "/login/", headers=headers, data=data, proxies=proxies, verify=False, allow_redirects=False)
  236. cookies = s.cookies.get_dict()
  237. if "location" in r.headers:
  238. if "/login/" in r.headers["location"]:
  239. print("[!] Login failed")
  240. sys.exit()
  241. if "ExpiredDomainssessid" in cookies:
  242. print("[+] Login successful. ExpiredDomainssessid: %s" % (cookies["ExpiredDomainssessid"]))
  243. else:
  244. print("[!] Login failed")
  245. sys.exit()
  246. def getIndex(cells, index):
  247. if cells[index].find("a") == None:
  248. return cells[index].text.strip()
  249. return cells[index].find("a").text.strip()
  250. ## MAIN
  251. if __name__ == "__main__":
  252. parser = argparse.ArgumentParser(
  253. description='Finds expired domains, domain categorization, and Archive.org history to determine good candidates for C2 and phishing domains',
  254. epilog = '''
  255. Examples:
  256. ./domainhunter.py -k apples -c --ocr -t5
  257. ./domainhunter.py --check --ocr -t3
  258. ./domainhunter.py --single mydomain.com
  259. ./domainhunter.py --keyword tech --check --ocr --timing 5 --alexa
  260. ./domaihunter.py --filename inputlist.txt --ocr --timing 5''',
  261. formatter_class=argparse.RawDescriptionHelpFormatter)
  262. parser.add_argument('-a','--alexa', help='Filter results to Alexa listings', required=False, default=0, action='store_const', const=1)
  263. parser.add_argument('-k','--keyword', help='Keyword used to refine search results', required=False, default=False, type=str, dest='keyword')
  264. parser.add_argument('-c','--check', help='Perform domain reputation checks', required=False, default=False, action='store_true', dest='check')
  265. parser.add_argument('-f','--filename', help='Specify input file of line delimited domain names to check', required=False, default=False, type=str, dest='filename')
  266. parser.add_argument('--ocr', help='Perform OCR on CAPTCHAs when challenged', required=False, default=False, action='store_true')
  267. parser.add_argument('-r','--maxresults', help='Number of results to return when querying latest expired/deleted domains', required=False, default=100, type=int, dest='maxresults')
  268. parser.add_argument('-s','--single', help='Performs detailed reputation checks against a single domain name/IP.', required=False, default=False, dest='single')
  269. parser.add_argument('-t','--timing', help='Modifies request timing to avoid CAPTCHAs. Slowest(0) = 90-120 seconds, Default(3) = 10-20 seconds, Fastest(5) = no delay', required=False, default=3, type=int, choices=range(0,6), dest='timing')
  270. parser.add_argument('-w','--maxwidth', help='Width of text table', required=False, default=400, type=int, dest='maxwidth')
  271. parser.add_argument('-V','--version', action='version',version='%(prog)s {version}'.format(version=__version__))
  272. parser.add_argument("-P", "--proxy", required=False, default=None, help="proxy. ex https://127.0.0.1:8080")
  273. parser.add_argument("-u", "--username", required=False, default=None, type=str, help="username for expireddomains.net")
  274. parser.add_argument("-p", "--password", required=False, default=None, type=str, help="password for expireddomains.net")
  275. parser.add_argument("-o", "--output", required=False, default=None, type=str, help="output file path")
  276. parser.add_argument('-ks','--keyword-start', help='Keyword starts with used to refine search results', required=False, default="", type=str, dest='keyword_start')
  277. parser.add_argument('-ke','--keyword-end', help='Keyword ends with used to refine search results', required=False, default="", type=str, dest='keyword_end')
  278. args = parser.parse_args()
  279. # Load dependent modules
  280. try:
  281. import requests
  282. from bs4 import BeautifulSoup
  283. from texttable import Texttable
  284. except Exception as e:
  285. print("Expired Domains Reputation Check")
  286. print("[-] Missing basic dependencies: {}".format(str(e)))
  287. print("[*] Install required dependencies by running `pip3 install -r requirements.txt`")
  288. quit(0)
  289. # Load OCR related modules if --ocr flag is set since these can be difficult to get working
  290. if args.ocr:
  291. try:
  292. import pytesseract
  293. from PIL import Image
  294. import shutil
  295. except Exception as e:
  296. print("Expired Domains Reputation Check")
  297. print("[-] Missing OCR dependencies: {}".format(str(e)))
  298. print("[*] Install required Python dependencies by running: pip3 install -r requirements.txt")
  299. print("[*] Ubuntu\Debian - Install tesseract by running: apt-get install tesseract-ocr python3-imaging")
  300. print("[*] macOS - Install tesseract with homebrew by running: brew install tesseract")
  301. quit(0)
  302. ## Variables
  303. username = args.username
  304. password = args.password
  305. proxy = args.proxy
  306. alexa = args.alexa
  307. keyword = args.keyword
  308. check = args.check
  309. filename = args.filename
  310. maxresults = args.maxresults
  311. single = args.single
  312. timing = args.timing
  313. maxwidth = args.maxwidth
  314. ocr = args.ocr
  315. output = args.output
  316. keyword_start = args.keyword_start
  317. keyword_end = args.keyword_end
  318. malwaredomainsURL = 'http://mirror1.malwaredomains.com/files/justdomains'
  319. expireddomainsqueryURL = 'https://www.expireddomains.net/domain-name-search'
  320. expireddomainHost = "https://member.expireddomains.net"
  321. timestamp = time.strftime("%Y%m%d_%H%M%S")
  322. useragent = 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)'
  323. headers = {'User-Agent':useragent}
  324. proxies = {}
  325. requests.packages.urllib3.disable_warnings()
  326. # HTTP Session container, used to manage cookies, session tokens and other session information
  327. s = requests.Session()
  328. if(args.proxy != None):
  329. proxy_parts = urlparse(args.proxy)
  330. proxies["http"] = "http://%s" % (proxy_parts.netloc)
  331. proxies["https"] = "https://%s" % (proxy_parts.netloc)
  332. title = '''
  333. ____ ___ __ __ _ ___ _ _ _ _ _ _ _ _ _____ _____ ____
  334. | _ \ / _ \| \/ | / \ |_ _| \ | | | | | | | | | \ | |_ _| ____| _ \
  335. | | | | | | | |\/| | / _ \ | || \| | | |_| | | | | \| | | | | _| | |_) |
  336. | |_| | |_| | | | |/ ___ \ | || |\ | | _ | |_| | |\ | | | | |___| _ <
  337. |____/ \___/|_| |_/_/ \_\___|_| \_| |_| |_|\___/|_| \_| |_| |_____|_| \_\ '''
  338. print(title)
  339. print("")
  340. print("Expired Domains Reputation Checker")
  341. print("Authors: @joevest and @andrewchiles\n")
  342. print("DISCLAIMER: This is for educational purposes only!")
  343. disclaimer = '''It is designed to promote education and the improvement of computer/cyber security.
  344. The authors or employers are not liable for any illegal act or misuse performed by any user of this tool.
  345. If you plan to use this content for illegal purpose, don't. Have a nice day :)'''
  346. print(disclaimer)
  347. print("")
  348. # Download known malware domains
  349. print('[*] Downloading malware domain list from {}\n'.format(malwaredomainsURL))
  350. maldomains = downloadMalwareDomains(malwaredomainsURL)
  351. maldomainsList = maldomains.split("\n")
  352. # Retrieve reputation for a single choosen domain (Quick Mode)
  353. if single:
  354. checkDomain(single)
  355. exit(0)
  356. # Perform detailed domain reputation checks against input file, print table, and quit. This does not generate an HTML report
  357. if filename:
  358. # Initialize our list with an empty row for the header
  359. data = []
  360. try:
  361. with open(filename, 'r') as domainsList:
  362. for line in domainsList.read().splitlines():
  363. data.append(checkDomain(line))
  364. doSleep(timing)
  365. # Print results table
  366. header = ['Domain', 'BlueCoat', 'IBM X-Force', 'Cisco Talos', 'MXToolbox']
  367. print(drawTable(header,data))
  368. except KeyboardInterrupt:
  369. print('Caught keyboard interrupt. Exiting!')
  370. exit(0)
  371. except Exception as e:
  372. print('[-] Error: {}'.format(e))
  373. exit(1)
  374. exit(0)
  375. # Lists for our ExpiredDomains results
  376. domain_list = []
  377. data = []
  378. # Generate list of URLs to query for expired/deleted domains
  379. urls = []
  380. if args.password == None or args.password == "":
  381. password = getpass.getpass("Password: ")
  382. loginExpiredDomains()
  383. m = 200
  384. if maxresults < m:
  385. m = maxresults
  386. for i in range (0,(maxresults),m):
  387. k=""
  388. if keyword:
  389. k=keyword
  390. urls.append('{}/domains/combinedexpired/?fwhois=22&fadult=1&start={}&ftlds[]=2&ftlds[]=3&ftlds[]=4&flimit={}&fdomain={}&fdomainstart={}&fdomainend={}&falexa={}'.format(expireddomainHost,i,m,k,keyword_start,keyword_end,alexa))
  391. max_reached = False
  392. for url in urls:
  393. print("[*] {}".format(url))
  394. domainrequest = s.get(url,headers=headers,verify=False,proxies=proxies)
  395. domains = domainrequest.text
  396. # Turn the HTML into a Beautiful Soup object
  397. soup = BeautifulSoup(domains, 'html.parser')
  398. try:
  399. table = soup.find_all("table", class_="base1")
  400. tbody = table[0].select("tbody tr")
  401. for row in tbody:
  402. # Alternative way to extract domain name
  403. # domain = row.find('td').find('a').text
  404. cells = row.findAll("td")
  405. if len(cells) == 1:
  406. max_reached = True
  407. break # exit if max rows reached
  408. if len(cells) >= 1:
  409. c0 = getIndex(cells, 0).lower() # domain
  410. c1 = getIndex(cells, 3) # bl
  411. c2 = getIndex(cells, 4) # domainpop
  412. c3 = getIndex(cells, 5) # birth
  413. c4 = getIndex(cells, 7) # Archive.org entries
  414. c5 = getIndex(cells, 8) # Alexa
  415. c6 = getIndex(cells, 10) # Dmoz.org
  416. c7 = getIndex(cells, 12) # status com
  417. c8 = getIndex(cells, 13) # status net
  418. c9 = getIndex(cells, 14) # status org
  419. c10 = getIndex(cells, 17) # status de
  420. c11 = getIndex(cells, 11) # TLDs
  421. c12 = getIndex(cells, 19) # RDT
  422. c13 = "" # List
  423. c14 = getIndex(cells, 22) # Status
  424. c15 = "" # links
  425. # create available TLD list
  426. available = ''
  427. if c7 == "available":
  428. available += ".com "
  429. if c8 == "available":
  430. available += ".net "
  431. if c9 == "available":
  432. available += ".org "
  433. if c10 == "available":
  434. available += ".de "
  435. # Only grab status for keyword searches since it doesn't exist otherwise
  436. status = ""
  437. if keyword:
  438. status = c14
  439. if keyword:
  440. # Only add Expired, not Pending, Backorder, etc
  441. # "expired" isn't returned any more, I changed it to "available"
  442. if c14 == "available": # I'm not sure about this, seems like "expired" isn't an option anymore. expireddomains.net might not support this any more.
  443. # Append parsed domain data to list if it matches our criteria (.com|.net|.org and not a known malware domain)
  444. if (c0.lower().endswith(".com") or c0.lower().endswith(".net") or c0.lower().endswith(".org")) and (c0 not in maldomainsList):
  445. domain_list.append([c0,c3,c4,available,status])
  446. # Non-keyword search table format is slightly different
  447. else:
  448. # Append original parsed domain data to list if it matches our criteria (.com|.net|.org and not a known malware domain)
  449. if (c0.lower().endswith(".com") or c0.lower().endswith(".net") or c0.lower().endswith(".org")) and (c0 not in maldomainsList):
  450. domain_list.append([c0,c3,c4,available,status])
  451. if max_reached:
  452. print("[*] All records returned")
  453. break
  454. except Exception as e:
  455. print("[!] Error: ", e)
  456. pass
  457. # Add additional sleep on requests to ExpiredDomains.net to avoid errors
  458. time.sleep(5)
  459. # Check for valid list results before continuing
  460. if len(domain_list) == 0:
  461. print("[-] No domain results found or none are currently available for purchase!")
  462. exit(0)
  463. else:
  464. domain_list_unique = []
  465. [domain_list_unique.append(item) for item in domain_list if item not in domain_list_unique]
  466. # Print number of domains to perform reputation checks against
  467. if check:
  468. print("\n[*] Performing reputation checks for {} domains".format(len(domain_list_unique)))
  469. print("")
  470. for domain_entry in domain_list_unique:
  471. domain = domain_entry[0]
  472. birthdate = domain_entry[1]
  473. archiveentries = domain_entry[2]
  474. availabletlds = domain_entry[3]
  475. status = domain_entry[4]
  476. bluecoat = '-'
  477. ibmxforce = '-'
  478. ciscotalos = '-'
  479. # Perform domain reputation checks
  480. if check:
  481. unwantedResults = ['Uncategorized','error','Not found.','Spam','Spam URLs','Pornography','badurl','Suspicious','Malicious Sources/Malnets','captcha','Phishing','Placeholders']
  482. bluecoat = checkBluecoat(domain)
  483. if bluecoat not in unwantedResults:
  484. print("[+] Bluecoat - {}: {}".format(domain, bluecoat))
  485. ibmxforce = checkIBMXForce(domain)
  486. if ibmxforce not in unwantedResults:
  487. print("[+] IBM XForce - {}: {}".format(domain, ibmxforce))
  488. ciscotalos = checkTalos(domain)
  489. if ciscotalos not in unwantedResults:
  490. print("[+] Cisco Talos {}: {}".format(domain, ciscotalos))
  491. print("")
  492. # Sleep to avoid captchas
  493. doSleep(timing)
  494. # Append entry to new list with reputation if at least one service reports reputation
  495. if not ((bluecoat in ('Uncategorized','badurl','Suspicious','Malicious Sources/Malnets','captcha','Phishing','Placeholders','Spam','error')) \
  496. and (ibmxforce in ('Not found.','error')) and (ciscotalos in ('Uncategorized','error'))):
  497. data.append([domain,birthdate,archiveentries,availabletlds,status,bluecoat,ibmxforce,ciscotalos])
  498. # Sort domain list by column 2 (Birth Year)
  499. sortedDomains = sorted(data, key=lambda x: x[1], reverse=True)
  500. if check:
  501. if len(sortedDomains) == 0:
  502. print("[-] No domains discovered with a desireable categorization!")
  503. exit(0)
  504. else:
  505. print("[*] {} of {} domains discovered with a potentially desireable categorization!".format(len(sortedDomains),len(domain_list)))
  506. # Build HTML Table
  507. html = ''
  508. htmlHeader = '<html><head><title>Expired Domain List</title></head>'
  509. htmlBody = '<body><p>The following available domains report was generated at {}</p>'.format(timestamp)
  510. htmlTableHeader = '''
  511. <table border="1" align="center">
  512. <th>Domain</th>
  513. <th>Birth</th>
  514. <th>Entries</th>
  515. <th>TLDs Available</th>
  516. <th>Status</th>
  517. <th>BlueCoat</th>
  518. <th>IBM X-Force</th>
  519. <th>Cisco Talos</th>
  520. <th>WatchGuard</th>
  521. <th>Namecheap</th>
  522. <th>Archive.org</th>
  523. '''
  524. htmlTableBody = ''
  525. htmlTableFooter = '</table>'
  526. htmlFooter = '</body></html>'
  527. # Build HTML table contents
  528. for i in sortedDomains:
  529. htmlTableBody += '<tr>'
  530. htmlTableBody += '<td>{}</td>'.format(i[0]) # Domain
  531. htmlTableBody += '<td>{}</td>'.format(i[1]) # Birth
  532. htmlTableBody += '<td>{}</td>'.format(i[2]) # Entries
  533. htmlTableBody += '<td>{}</td>'.format(i[3]) # TLDs
  534. htmlTableBody += '<td>{}</td>'.format(i[4]) # Status
  535. htmlTableBody += '<td><a href="https://sitereview.bluecoat.com/" target="_blank">{}</a></td>'.format(i[5]) # Bluecoat
  536. htmlTableBody += '<td><a href="https://exchange.xforce.ibmcloud.com/url/{}" target="_blank">{}</a></td>'.format(i[0],i[6]) # IBM x-Force Categorization
  537. htmlTableBody += '<td><a href="https://www.talosintelligence.com/reputation_center/lookup?search={}" target="_blank">{}</a></td>'.format(i[0],i[7]) # Cisco Talos
  538. htmlTableBody += '<td><a href="http://www.borderware.com/domain_lookup.php?ip={}" target="_blank">WatchGuard</a></td>'.format(i[0]) # Borderware WatchGuard
  539. htmlTableBody += '<td><a href="https://www.namecheap.com/domains/registration/results.aspx?domain={}" target="_blank">Namecheap</a></td>'.format(i[0]) # Namecheap
  540. htmlTableBody += '<td><a href="http://web.archive.org/web/*/{}" target="_blank">Archive.org</a></td>'.format(i[0]) # Archive.org
  541. htmlTableBody += '</tr>'
  542. html = htmlHeader + htmlBody + htmlTableHeader + htmlTableBody + htmlTableFooter + htmlFooter
  543. logfilename = "{}_domainreport.html".format(timestamp)
  544. if output != None:
  545. logfilename = output
  546. log = open(logfilename,'w')
  547. log.write(html)
  548. log.close
  549. print("\n[*] Search complete")
  550. print("[*] Log written to {}\n".format(logfilename))
  551. # Print Text Table
  552. header = ['Domain', 'Birth', '#', 'TLDs', 'Status', 'BlueCoat', 'IBM', 'Cisco Talos']
  553. print(drawTable(header,sortedDomains))