关于FOFA的几个脚本

说明

FOFA的一些使用脚本

FOFA API Demo

pip install fofa

Windows环境
需要 用户名/密码 存储于环境变量中

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2017/10/7 19:53
# @Author  : BLKStone
# @Site    :
# @File    : demo_search.py.py
# @Software: PyCharm Community Edition

import fofa
import os

def fofa_search():
    target_ip_file_path = 'iplist.txt'
    email, key = (str(os.environ['FOFA_USERNAME']), str(os.environ['FOFA_API_KEY']))  # 输入email和key
    client = fofa.Client(email, key)  # 将email和key传入fofa.Client类进行初始化和验证，并得到一个fofa client对象
    query_str = 'title="F420" && port="80"'
    fcoin = client.get_userinfo()["fcoin"]  # 查询F币剩余数量
    print "查询前 fcoin:", fcoin

    with open(target_ip_file_path, 'w') as f:
        for page in range(1, 5):
            # 从第1页查到第50页         #
            # 当F币剩249个时，不再获取数据
            print "page:", page
            try:
                data = client.get_data(query_str, page=page, fields="ip,city")  # 查询第page页数据的ip和城市
                for ip, city in data["results"]:
                    print "%s,%s" % (ip, city)  # 打印出每条数据的ip和城市
                    f.write(str(ip) + '\n')
            except Exception, e:
                print ''
    f.close()

    fcoin = client.get_userinfo()["fcoin"]
    print "查询后 fcoin:", fcoin


def check_credential():
    print "Username:", os.environ['FOFA_USERNAME']
    print "API_KEY:", os.environ['FOFA_API_KEY']

if __name__ == "__main__":
    check_credential()
    fofa_search()

FOFA Selenium 脚本

Selenium

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2017/10/8 1:03
# @Author  : BLKStone
# @Site    : http://blkstone.github.io
# @File    : bs4_search
# @Software: PyCharm Community Edition

# http://selenium-python.readthedocs.io/waits.html
# https://www.crummy.com/software/BeautifulSoup/bs4/doc/#find
# Selenium send_keys utf-8 中文问题
# http://blog.csdn.net/u012113628/article/details/56840608


import sys
import requests
import os
import time
import base64

from bs4 import BeautifulSoup

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# reload(sys)
# sys.setdefaultencoding('utf-8')


def send_query(query_keyword):
    base64_query_keyword = base64.standard_b64encode(query_keyword)
    page = 1
    uri = build_fofa_query_uri(base64_query_keyword, page)
    print uri

def build_fofa_query_uri(base64_query_keyword, page):
    base_uri = 'https://fofa.so'
    uri_parameter_pattern = '/result?page='+str(page)+'&qbase64='+ base64_query_keyword + '&full=true'
    uri = base_uri + uri_parameter_pattern
    return uri

# 等待页面 ajax 加载
def wait_for_ajax_loading(expected_condition, driver):
    element = None
    try:
        element = WebDriverWait(driver, 600).until(expected_condition)
    except Exception,e:
        print "等待 list_mod 超时，退出浏览器。"
        driver.quit()

# 登陆认证
def fofa_login(driver):
    username = os.environ['FOFA_USERNAME']
    password = os.environ['FOFA_PASSWORD']

    username_input = driver.find_element_by_id('username')
    pwd_input = driver.find_element_by_id('password')
    login_btn = driver.find_element_by_xpath('//*[@id="login-form"]/table/tbody/tr[4]/td/button')
    username_input.send_keys(username)
    time.sleep(1)
    pwd_input.send_keys(password)
    login_btn.submit()

# 获取结果总数
def extract_result_number_from_list_jg(list_jg):
    # print list_jg.contents
    # for idx,ele in enumerate(list_jg.contents):
    #     print idx,ele
    pre_idx = list_jg.contents[2].find('获得'.decode('utf-8'))+3
    suf_idx = list_jg.contents[2].find('条匹'.decode('utf-8'))
    number = list_jg.contents[2][pre_idx:suf_idx].strip()
    print "结果数量：", number
    return number

# 单页数据获取
def single_page_extract(divs, page):
    page_items = []
    for idx, div in enumerate(divs):
        # print div.div.a
        # print div.div.a['href']
        print '编号:', idx + (page-1)*10
        print 'URL:', div.div.a.contents[0].strip()
        list_mod_c = div.find('div', class_='list_mod_c')
        item = extract_list_mod_c(list_mod_c)
        item['URL'] = div.div.a.contents[0].strip()
        page_items.append(item)
    return page_items

def next_page_extract(driver, page):
    next_page_button = driver.find_element_by_class_name('next_page')
    next_page_button.click()
    wait_for_ajax_loading(EC.presence_of_element_located((By.CLASS_NAME, "list_mod")), driver)
    time.sleep(2)
    whole_page = driver.find_element_by_xpath('/html')
    page_html = whole_page.get_attribute("outerHTML")
    soup = BeautifulSoup(page_html, 'html.parser')
    divs = soup.find_all("div", class_="list_mod")
    return single_page_extract(divs, page)


# {
# 'City': u'Quzhou',
# 'Domain': u'qzmzj.gov.cn',
# 'Title': u'\u8862\u5dde\u798f\u5229\u5f69\u7968\u7f51',
# 'URL': u'fc.qzmzj.gov.cn',
# 'IP': u'61.130.66.250',
# 'Country': u'China',
# 'Date': u'2017-10-03',
# 'OS': u'centos'
# }
def page_items_process(page_items, f):
    for item in page_items:
        f.write(str(item['IP'])+'\n')

def webdriver_action():
    output_path = 'F420_list.txt'

    webdriver_path = 'C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe'
    driver = webdriver.Chrome(webdriver_path)  # Optional argument, if not specified will search path.
    driver.get('https://i.nosec.org/login?service=http%3A%2F%2Ffofa.so%2Fusers%2Fservice')

    # login
    fofa_login(driver)

    # search
    search_box = driver.find_element_by_id('q')
    # search_box.send_keys('title="彩" && host=".gov.cn"')
    search_box.send_keys('title="F420"'.decode('utf-8'))
    time.sleep(1)
    search_btn = driver.find_element_by_xpath('//*[@id="search_form"]/input[2]')
    search_btn.click()

    # 等待加载
    wait_for_ajax_loading(EC.presence_of_element_located((By.CLASS_NAME, "list_mod")), driver)

    whole_page = driver.find_element_by_xpath('/html')
    page_html = whole_page.get_attribute("outerHTML")
    soup = BeautifulSoup(page_html, 'html.parser')

    # 获取结果总数
    list_jg = soup.find('div', class_='list_jg')
    extract_result_number_from_list_jg(list_jg)
    page = 1

    with open(output_path,'w') as f:
        # 单页数据获取
        divs = soup.find_all("div", class_="list_mod")
        page_items = single_page_extract(divs, page)
        page_items_process(page_items,f)
        # 后续页面数据获取
        for page in range(2,111):
            page_items = next_page_extract(driver, page)
            page_items_process(page_items, f)
            time.sleep(5)
    f.close()
    # driver.implicitly_wait()
    # driver.quit()

# 存在bug的数据读取
# title="彩" && host=".gov.cn"
# 湖州福彩网
def extract_list_mod_c_2(list_mod_c):
    lis_iter = list_mod_c.find_all('li')
    lis = []
    for li in lis_iter:
        lis.append(li)
    # print lis

    try:
        data_title = lis[0].contents[1]
    except Exception,e:
        print e
        data_title = 'N/A'

    try:
        data_os = lis[1].contents[1].string
    except Exception,e:
        print e
        data_os = 'N/A'

    try:
        data_ip = lis[2].contents[2].string
    except Exception,e:
        print e
        data_ip = 'N/A'

    try:
        data_date = lis[3].contents[1]
    except Exception,e:
        print e
        data_date = 'N/A'

    try:
        data_country = lis[4].contents[2].a.string
    except Exception,e:
        print e
        data_country = 'N/A'

    try:
        data_city = lis[4].contents[2].a.find_next_sibling('a').string
    except Exception,e:
        print e
        data_city = 'N/A'

    try:
        data_domain = lis[5].contents[2].string
    except Exception,e:
        print e
        data_domain = 'N/A'

    print 'Title:', data_title
    print 'OS:', data_os
    print 'IP:', data_ip
    print 'Date:', data_date
    print 'Country:', data_country
    print 'City:', data_city
    print 'Domain:', data_domain

# 解析单页数据
def extract_list_mod_c(list_mod_c):
    lis_iter = list_mod_c.find_all('li')

    # icon_mapping = {
    #     'fa fa-cog':'OS',
    #     'fa fa-map-marker':'IP',
    #     'fa fa-clock-o':'Date',
    #     'fa fa-plane':'Country/City',
    #     'fa fa-leaf':'Domain',
    # }

    result = {
        'Title':'N/A',
        'OS':'N/A',
        'IP':'N/A',
        'Date':'N/A',
        'Country':'N/A',
        'City':'N/A',
        'Domain':'N/A',
    }

    for idx, li in enumerate(lis_iter):
        if idx == 0:
            result['Title'] = li.contents[1].strip()
        else:
            # li.i.get_attribute('class') 为了获取标签(tag)的属性值(attribute)，这种写法并不行
            if li.i is not None:
                if 'fa-cog' in li.i.attrs['class']:
                    result['OS'] = li.a.string
                elif 'fa-map-marker' in li.i.attrs['class']:
                    result['IP'] = li.a.string
                elif 'fa-clock-o' in li.i.attrs['class']:
                    result['Date'] = li.contents[1].strip()
                elif 'fa-plane' in li.i.attrs['class']:
                    if li.a is None:
                        result['Country'] = 'N/A'
                        result['City'] = 'N/A'
                    else:
                        result['Country'] = li.a.string
                        result['City'] = li.a.find_next_sibling('a').string
                elif 'fa-leaf' in li.i.attrs['class']:
                    result['Domain'] = li.a.string
                else:
                    pass
            else:
                pass

    print 'Title:', result['Title']
    print 'OS:', result['OS']
    print 'IP:', result['IP']
    print 'Date:', result['Date']
    print 'Country:', result['Country']
    print 'City:', result['City']
    print 'Domain:', result['Domain']

    return result



if __name__ == '__main__':
    # send_query('title="F420"')
    webdriver_action()