diff --git a/.secret b/.secret deleted file mode 100644 index 38e155a..0000000 --- a/.secret +++ /dev/null @@ -1 +0,0 @@ -ontipj0py431xul5zm4kj6gzzo|3600|1715933457.543795|oadugxyl9fhoqsamqopc \ No newline at end of file diff --git a/decspider/items.py b/decspider/items.py index 5be5d76..c44adb5 100644 --- a/decspider/items.py +++ b/decspider/items.py @@ -15,7 +15,6 @@ class NewsItem(scrapy.Item): source = scrapy.Field() content = scrapy.Field() image_urls = scrapy.Field() - classify = scrapy.Field() collection = scrapy.Field() url = scrapy.Field() source_url = scrapy.Field() diff --git a/decspider/myutils.py b/decspider/myutils.py index d0b7d2d..84206dc 100644 --- a/decspider/myutils.py +++ b/decspider/myutils.py @@ -5,10 +5,7 @@ import requests, datetime from .settings import SECRETID, SECRETKEY, PROXYPOOL_UPDATENUM, PROXYPOOL_MIN_NUM, PROXYPOOL_MIN_DURATION -SECRET_PATH = './.secret' - - -def _get_secret_token(): +def get_secret_token(): r = requests.post(url='https://auth.kdlapi.com/api/get_secret_token', data={'secret_id': SECRETID, 'secret_key': SECRETKEY}) if r.status_code != 200: raise KdlException(r.status_code, r.content.decode('utf8')) @@ -22,31 +19,9 @@ def _get_secret_token(): return secret_token, expire, _time -def _read_secret_token(): - with open(SECRET_PATH, 'r') as f: - token_info = f.read() - secret_token, expire, _time, last_secret_id = token_info.split('|') - if float(_time) + float(expire) - 3 * 60 < time.time() or SECRETID != last_secret_id: # 还有3分钟过期或SecretId变化时更新 - secret_token, expire, _time = _get_secret_token() - with open(SECRET_PATH, 'w') as f: - f.write(secret_token + '|' + expire + '|' + _time + '|' + SECRETID) - return secret_token - - -def get_secret_token(): - if os.path.exists(SECRET_PATH): - secret_token = _read_secret_token() - else: - secret_token, expire, _time = _get_secret_token() - with open(SECRET_PATH, 'w') as f: - f.write(secret_token + '|' + expire + '|' + _time + '|' + SECRETID) - return secret_token - - class KdlException(Exception): """异常类""" - def __init__(self, code=None, message=None): self.code = code if sys.version_info[0] < 3 and isinstance(message, unicode):