Python爬虫自动化获取华图和粉笔网站的错题(推荐)

##导包
import requests
import lxml.etree
import re
import time
import os
from docx import Document
from docx.shared import Inches
from docx.shared import Pt
from docx.shared import Inches
from docx.oxml.ns import qn
from docx.enum.text import WD_ALIGN_PARAGRAPH
url = str(input("请输入练习的网址："))
headers={
###完整的headers，否则获取不到数据
}
response = requests.get(url = url,headers = headers)
response.encoding='utf-8'
reptext = response.text
tree = lxml.etree.HTML(reptext) #解析网站获取源码

dirName="考公图片"
if not os.path.exists(dirName):
 os.mkdir(dirName) #网站图片保存路径
 
jiexi = re.findall(r'<div class="jiexi-item-title">解析.*"(.*".*?>', each)) #获取解析里的图片URL
 
imgt = []
for each in imgg:
 if each == []:
  imgt.append([1])
 else:
  imgt.append(each) #把解析里图片URL美化整理一下
  
jiexilast = []
for qq in jiexi:
 jiexilast.append(re.sub(r'<[^>]+>', '', qq)) #美化题目解析
 
corrected = re.findall(r'<span class="g-right-answer-color">[a-zA-Z]{1,4}</span>', reptext) #获取正确答案
correct = []
for ee in corrected:
 correct.append(re.sub(r'<[^>]+>', '', ee)) #美化正确答案
 
yoursed = re.findall(r'<span class="yellowWord">[a-zA-Z]{1,4}</span>', reptext) #获取自己的答案
yours = []
for ee in yoursed:
 yours.append(re.sub(r'<[^>]+>', '', ee)) #美化自己的答案
 
timuleixing = re.findall(r'<span class="greenWord">(.*"greenWord">.*"(.*".*?>', each))
imgx = []
for each in img:
 if each == []:
  imgx.append([1])
 else:
  imgx.append(each) #最终版题目图片URL
  

v = tree.xpath('//div[@class="exercise-main-title"]//text()') #本次题目类型

try:
 ###这是既有复合题也有单选题的
 fuheti = re.findall(r'<!--复合题-->(.*"exercise-main-topics"',reptext,re.S)[0].split('<!--复合题-->')
except:
 try:
  ###这是只有复合题或者复合题在最后几题的
  fuheti = re.findall(r'<!--复合题-->(.*"main-topic-choices">(.*"main-topic-letters clearfix pl14">',reptext,re.S)
for everything in xuanxiang:
 try: ##处理只有两个选项
  axuanxiang.append(re.sub("<.*?>","",re.findall(r'<div.*"<.*?>","",re.findall(r'<div.*"<.*?>","",re.findall(r'<div.*"<.*?>","",re.findall(r'<div.*"(.*".*?>',re.findall(r'.*"保存成功")
       document.add_picture(img_path, width=Inches(5))
      choose.append(fuheti_last)
  except:
   pass
  
  ###导入单选题题目
  p = document.add_paragraph()
  run = p.add_run(str(count+1)+"、"+timuleixing[count]+find5[count][3:])
  run.font.size = Pt(14)
  run.font.name=u'宋体'
  r = run._element
  r.rPr.rFonts.set(qn('w:eastAsia'),u'宋体')
  url = imgx[count][0]
  headers ={
   'Use-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36'
  }
  try:
   img_data = requests.get(url = url,headers = headers).content
   img_path = dirName+'/'+'tupian'+'.jpg'
   with open(img_path,'wb') as fp:
    fp.write(img_data)
    print("保存成功")
   document.add_picture(img_path, width=Inches(5))
   count+=1
  except:
   count+=1
   
  ###导入选项
  p = document.add_paragraph()
  run = p.add_run(axuanxiang[count-1])
  run.font.size = Pt(14)
  run.font.name=u'宋体'
  r = run._element
  r.rPr.rFonts.set(qn('w:eastAsia'),u'宋体')
  p = document.add_paragraph()
  run = p.add_run(bxuanxiang[count-1])
  run.font.size = Pt(14)
  run.font.name=u'宋体'
  r = run._element
  r.rPr.rFonts.set(qn('w:eastAsia'),u'宋体')
  p = document.add_paragraph()
  run = p.add_run(cxuanxiang[count-1])
  run.font.size = Pt(14)
  run.font.name=u'宋体'
  r = run._element
  r.rPr.rFonts.set(qn('w:eastAsia'),u'宋体')
  p = document.add_paragraph()
  run = p.add_run(dxuanxiang[count-1])
  run.font.size = Pt(14)
  run.font.name=u'宋体'
  r = run._element
  r.rPr.rFonts.set(qn('w:eastAsia'),u'宋体')
  p = document.add_paragraph()
  run = p.add_run("\n")
  run.font.size = Pt(14)
  run.font.name=u'宋体'
  r = run._element
  r.rPr.rFonts.set(qn('w:eastAsia'),u'宋体')
  
 else:
  count+=1

###美化界面
p = document.add_paragraph()
run = p.add_run("\n\n\n\n\n")
run.font.size = Pt(14)
run.font.name=u'宋体'
r = run._element
r.rPr.rFonts.set(qn('w:eastAsia'),u'宋体')

###美化解析
counting = 0
jiexilast2 = []
for ok in jiexilast:
 jiexilast2.append(re.sub(r'\n\t\t','：',ok))
for every in correct:
 if every != yours[counting]:
  ###导入解析和答案
  p = document.add_paragraph()
  run = p.add_run(str(counting+1)+"、"+"正确答案为："+correct[counting]+"\n"+jiexilast2[counting])
  run.font.size = Pt(14)
  run.font.name=u'宋体'
  r = run._element
  r.rPr.rFonts.set(qn('w:eastAsia'),u'宋体')
  url = imgt[counting][0]
  headers ={
   'Use-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36'
  }
  try:
   img_data = requests.get(url = url,headers = headers).content
   img_path = dirName+'/'+'tupian'+'.jpg'
   with open(img_path,'wb') as fp:
    fp.write(img_data)
    print("保存成功")
   document.add_picture(img_path, width=Inches(5))
   print("写入成功")
   counting+=1
  except:
   counting+=1
 else:
  counting+=1
###保存文档
document.save(v[0][5:-3]+'.docx')
print(v[0][5:-3]+'保存成功！')