cooperchen/newsspider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove .secret file read and write

Browse Source
This commit is contained in:
cooper 2024-05-17 16:33:13 +08:00
parent 2903506991
commit 4809e42fa7
3 changed files with 1 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.secret
View File

@ -1 +0,0 @@
ontipj0py431xul5zm4kj6gzzo|3600|1715933457.543795|oadugxyl9fhoqsamqopc

1
decspider/items.py
View File

@ -15,7 +15,6 @@ class NewsItem(scrapy.Item):
source = scrapy.Field() source = scrapy.Field()
content = scrapy.Field() content = scrapy.Field()
image_urls = scrapy.Field() image_urls = scrapy.Field()
classify = scrapy.Field()
collection = scrapy.Field() collection = scrapy.Field()
url = scrapy.Field() url = scrapy.Field()
source_url = scrapy.Field() source_url = scrapy.Field()

27
decspider/myutils.py
View File

@ -5,10 +5,7 @@ import requests, datetime
from .settings import SECRETID, SECRETKEY, PROXYPOOL_UPDATENUM, PROXYPOOL_MIN_NUM, PROXYPOOL_MIN_DURATION from .settings import SECRETID, SECRETKEY, PROXYPOOL_UPDATENUM, PROXYPOOL_MIN_NUM, PROXYPOOL_MIN_DURATION
SECRET_PATH = './.secret' def get_secret_token():
def _get_secret_token():
r = requests.post(url='https://auth.kdlapi.com/api/get_secret_token', data={'secret_id': SECRETID, 'secret_key': SECRETKEY}) r = requests.post(url='https://auth.kdlapi.com/api/get_secret_token', data={'secret_id': SECRETID, 'secret_key': SECRETKEY})
if r.status_code != 200: if r.status_code != 200:
raise KdlException(r.status_code, r.content.decode('utf8')) raise KdlException(r.status_code, r.content.decode('utf8'))
@ -22,31 +19,9 @@ def _get_secret_token():
return secret_token, expire, _time return secret_token, expire, _time
def _read_secret_token():
with open(SECRET_PATH, 'r') as f:
token_info = f.read()
secret_token, expire, _time, last_secret_id = token_info.split('|')
if float(_time) + float(expire) - 3 * 60 < time.time() or SECRETID != last_secret_id: # 还有3分钟过期或SecretId变化时更新
secret_token, expire, _time = _get_secret_token()
with open(SECRET_PATH, 'w') as f:
f.write(secret_token + '|' + expire + '|' + _time + '|' + SECRETID)
return secret_token
def get_secret_token():
if os.path.exists(SECRET_PATH):
secret_token = _read_secret_token()
else:
secret_token, expire, _time = _get_secret_token()
with open(SECRET_PATH, 'w') as f:
f.write(secret_token + '|' + expire + '|' + _time + '|' + SECRETID)
return secret_token
class KdlException(Exception): class KdlException(Exception):
"""异常类""" """异常类"""
def __init__(self, code=None, message=None): def __init__(self, code=None, message=None):
self.code = code self.code = code
if sys.version_info[0] < 3 and isinstance(message, unicode): if sys.version_info[0] < 3 and isinstance(message, unicode):