xray+rad 自动化edusrc

最近挖洞效率低下，学习了下扫描器

1.爬取漏洞平台上各大学校的网址

写了个小爬虫，有些会爬到百度百科里面，但是不影响，后面加个判断就好了。

import requests
import os
from lxml import etree
allpage = 199
all_school = dict()
#获取学校名字
handers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36 Edg/97.0.1072.76'}
print("开始获取学校名称和网址")
for page in range(1,allpage):
    #获取学校名称
    url = f'https://src.sjtu.edu.cn/rank/firm/?page={page}'
    page_text = requests.get(url,headers=handers).text
    tree = etree.HTML(page_text)
    content = tree.xpath(f'//tr[@class="row"]//a//text()')
    #定义一个进度条用的东西
    index = 1
    #从bing上获取学校官网
    for item in content:
        try:
            response = requests.get(url=f'https://cn.bing.com/search?q={item}', headers=handers).text
            tree2 = etree.HTML(response)
            content2 = tree2.xpath(f'/html/body/div[1]/main/ol/li[1]/div[1]/h2/a/@href')
            #把结果存入字典
            all_school[item]=content2[0]
            print(item + f'  {content2[0]}  '+f'第{page}/{allpage}页----{index/15*100}%')
            index += 1
        except:
            print(f"{item}出错啦！")
            index += 1
#持久化存储
with open('./学校网址.txt',mode='w',encoding='utf-8') as fp:
    for key in all_school.keys():
        fp.write(f'{key}    {all_school[key]}\n')
#单独存储url方便扫描
with open('url.txt',mode='w',encoding='utf-8') as fp2:
    for value in all_school.values():
        fp2.write(f'{value}\n')