147 lines
4.5 KiB
Python
147 lines
4.5 KiB
Python
|
#!/usr/bin/python
|
|||
|
|
# -*- coding:utf-8 -*-
|
|||
|
|
|
|||
|
|
import requests # 用来抓取网页的html源码
|
|||
|
|
import random # 取随机数
|
|||
|
|
from bs4 import BeautifulSoup # 用于代替正则式 取源码中相应标签中的内容
|
|||
|
|
import sys
|
|||
|
|
import time # 时间相关操作
|
|||
|
|
import js2py
|
|||
|
|
import os
|
|||
|
|
from tqdm import tqdm
|
|||
|
|
|
|||
|
|
|
|||
|
|
class downloader(object):
|
|||
|
|
def __init__(self):
|
|||
|
|
self.server = 'https://www.iimanhua.cc/'
|
|||
|
|
self.target = 'https://www.iimanhua.cc/comic/2189/'
|
|||
|
|
self.names = [] # 章节名
|
|||
|
|
self.urls = [] # 章节链接
|
|||
|
|
self.nums = 0 # 章节数
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
获取html文档内容
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def get_content(self, url):
|
|||
|
|
# 设置headers是为了模拟浏览器访问 否则的话可能会被拒绝 可通过浏览器获取，这里不用修改
|
|||
|
|
header = {
|
|||
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|||
|
|
'Connection': 'keep-alive',
|
|||
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|||
|
|
'Accept-Language': 'zh-CN, zh',
|
|||
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# 设置一个超时时间 取随机数 是为了防止网站被认定为爬虫，不用修改
|
|||
|
|
timeout = random.choice(range(80, 180))
|
|||
|
|
|
|||
|
|
while True:
|
|||
|
|
try:
|
|||
|
|
req = requests.get(url=url, headers=header)
|
|||
|
|
req.encoding = 'GBK' #这里是网页的编码转换，根据网页的实际需要进行修改，经测试这个编码没有问题
|
|||
|
|
break
|
|||
|
|
except Exception as e:
|
|||
|
|
print('3', e)
|
|||
|
|
time.sleep(random.choice(range(5, 10)))
|
|||
|
|
return req.text
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
获取下载的章节目录
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def get_download_catalogue(self, url):
|
|||
|
|
# html = self.get_content(url)
|
|||
|
|
# bf = BeautifulSoup(html, 'html.parser')
|
|||
|
|
# print(bf)
|
|||
|
|
# texts = bf.find_all('div', {'class': 'listmain'})
|
|||
|
|
|
|||
|
|
finename = "./kkk.txt"
|
|||
|
|
|
|||
|
|
f = open(finename,'r', encoding='utf-8') # 返回一个文件对象
|
|||
|
|
line = f.readline()
|
|||
|
|
while line:
|
|||
|
|
# print(line.strip('\n'))
|
|||
|
|
name, url = self.get_url(line)
|
|||
|
|
self.names.append(name)
|
|||
|
|
self.urls.append(self.server + url)
|
|||
|
|
line = f.readline()
|
|||
|
|
# print(self.urls)
|
|||
|
|
self.nums = len(self.urls)
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
获取下载的具体章节
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def get_url(self, url_str):
|
|||
|
|
st = url_str.find("/comic")
|
|||
|
|
ed = url_str.find("\" title")
|
|||
|
|
st2 = url_str.find("）")
|
|||
|
|
ed2 = url_str.find("\">")
|
|||
|
|
url = url_str[st:ed]
|
|||
|
|
name = url_str[st2+1:ed2]
|
|||
|
|
return name, url
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
def get_download_content(self, chap, path, name, url):
|
|||
|
|
#html = self.get_content(url)
|
|||
|
|
chappath = os.path.join(path, str(chap).zfill(3)+ '话 ' +name)
|
|||
|
|
os.makedirs(chappath, exist_ok=True)
|
|||
|
|
html = self.get_content(url)
|
|||
|
|
bf = BeautifulSoup(html, 'html.parser')
|
|||
|
|
jscmd = bf.find('script', {'language': 'javascript', 'type': 'text/javascript'}).text
|
|||
|
|
# print(jscmd)
|
|||
|
|
jscmd += '''
|
|||
|
|
\nvar b=base64decode(packed).slice(4);
|
|||
|
|
var a = eval(b);
|
|||
|
|
'''
|
|||
|
|
# print(jscmd)
|
|||
|
|
jsres = js2py.eval_js(jscmd) #执行js代码
|
|||
|
|
imgurls = self.get_img_url(jsres)
|
|||
|
|
page_no = 1
|
|||
|
|
for imgurl in tqdm(imgurls):
|
|||
|
|
r=requests.get(imgurl)
|
|||
|
|
with open(chappath+'/'+str(page_no)+'.jpg','wb') as f:
|
|||
|
|
f.write(r.content) #写入二进制内容
|
|||
|
|
page_no += 1
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
解析url
|
|||
|
|
"""
|
|||
|
|
def get_img_url(self, jsres):
|
|||
|
|
imgserver = 'https://res.img.96youhuiquan.com/'
|
|||
|
|
imgstrs = jsres.split(";")
|
|||
|
|
imgurls = []
|
|||
|
|
for imgstr in imgstrs:
|
|||
|
|
if len(imgstr)>1:
|
|||
|
|
st = imgstr.find("]=")
|
|||
|
|
imgurl = imgstr[st+3:-1]
|
|||
|
|
imgurls.append(imgserver+imgurl)
|
|||
|
|
return imgurls
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
def writer(self, path, name, text):
|
|||
|
|
write_flag = True
|
|||
|
|
with open(path, 'a', encoding='utf-8') as f:
|
|||
|
|
f.writelines(name)
|
|||
|
|
f.write('\n')
|
|||
|
|
f.writelines(text)
|
|||
|
|
f.write('\n\n')
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
path = './duannao/'
|
|||
|
|
dl = downloader()
|
|||
|
|
dl.get_download_catalogue(dl.target)
|
|||
|
|
for chap_no in range(77-1, dl.nums):
|
|||
|
|
print("第" + str(chap_no+1) + "话")
|
|||
|
|
dl.get_download_content(chap_no+1, path, dl.names[chap_no], dl.urls[chap_no])
|