迁移到playwright；其他微调

yjqiang · yjqiang · commit 46abf629558d · 2021-03-07T19:49:52.000+08:00
diff --git a/aiohttp_websession.py b/aiohttp_websession.py
@@ -0,0 +1,83 @@
+import sys
+import asyncio
+from typing import Any
+
+import aiohttp
+
+import printer
+
+sem = asyncio.Semaphore(3)
+
+
+class WebSession:
+    __slots__ = ('session',)
+
+    DEFAULT_OK_STATUS_CODES = (200,)
+    DEFAULT_IGNORE_STATUS_CODES = ()
+
+    def __init__(self):
+        self.session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=8))
+
+    @staticmethod
+    async def _recv_json(rsp: aiohttp.ClientResponse):
+        return await rsp.json(content_type=None)
+
+    @staticmethod
+    async def _recv_str(rsp: aiohttp.ClientResponse):
+        return await rsp.text()
+
+    @staticmethod
+    async def _recv_bytes(rsp: aiohttp.ClientResponse):
+        return await rsp.read()
+
+    @staticmethod
+    async def _recv(rsp: aiohttp.ClientResponse):
+        return rsp
+
+    # 基本就是通用的 request
+    async def _orig_req(self, parse_rsp, method, url, keep_try=True, **kwargs):
+        i = 0
+        while keep_try or i == 0:
+            i += 1
+            if i >= 10:
+                printer.warn(f'反复请求多次未成功, {url}, {kwargs}')
+                await asyncio.sleep(1)
+            try:
+                async with self.session.request(method, url, **kwargs) as rsp:
+                    if rsp.status == 200:
+                        body = await parse_rsp(rsp)
+                        if body:
+                            return body
+            except asyncio.CancelledError:
+                raise
+            except:
+                # print('当前网络不好，正在重试，请反馈开发者!!!!')
+                print(sys.exc_info()[0], sys.exc_info()[1], url)
+
+    async def request_json(self,
+                           method,
+                           url,
+                           keep_try=True,
+                           **kwargs) -> Any:
+        return await self._orig_req(self._recv_json, method, url, keep_try, **kwargs)
+
+    async def request_binary(self,
+                             method,
+                             url,
+                             keep_try=True,
+                             **kwargs) -> bytes:
+        return await self._orig_req(self._recv_bytes, method, url, keep_try, **kwargs)
+
+    async def request_text(self,
+                           method,
+                           url,
+                           keep_try=True,
+                           **kwargs) -> str:
+        return await self._orig_req(self._recv_str, method, url, keep_try, **kwargs)
+
+    # 返回 response
+    async def request(self,
+                      method,
+                      url,
+                      **kwargs) -> aiohttp.ClientResponse:
+        return await self._orig_req(self._recv, method, url, **kwargs)
diff --git a/client.py b/client.py
@@ -1,28 +1,30 @@
-import time
 import base64
 from urllib import parse
+import asyncio
+from typing import Optional, Tuple, Any
+import time
 
 import rsa
 
 from printer import info as print
-from web_session import WebSession
+from aiohttp_websession import WebSession
 from client_core import utils
 
 
 class Bili:
-    def __init__(self):
+    def __init__(self, web_session: WebSession):
         dict_user = utils.get_1st_user('client_core/conf/user.toml')
         dict_bili = utils.get_dict_bili('client_core/conf/bili.toml')
         self.user = utils.new_user(dict_user, dict_bili)
 
         self.name = self.user.name
         self.password = self.user.password
-        self.__web_session = WebSession()
+        self._web_session = web_session
 
-    def get_key(self):
+    async def get_key(self) -> Tuple[str, str]:
         url = f'https://passport.bilibili.com/api/oauth2/getKey'
         params = self.user.sort_and_sign()
-        json_rsp = self.__web_session.request_json(
+        json_rsp = await self._web_session.request_json(
             'POST', url, headers=self.user.dict_bili['appheaders'], params=params)
 
         data = json_rsp['data']
@@ -37,61 +39,66 @@ def get_key(self):
 
         return url_password, url_name
 
-    def login(self, url_password, url_name, seccode, challenge, validate):
-
+    async def login(self, url_password: str, url_name: str, seccode: str, challenge: str, validate: str) -> Optional[str]:
         extra_params = [
             # f'captcha=',
             f'password={url_password}',
             f'username={url_name}',
             f'ts={utils.curr_time()}',
-            f'seccode={"" if not seccode else parse.quote_plus(seccode)}',
-            f'challenge={"" if not challenge else parse.quote_plus(challenge)}',
-            f'validate={"" if not validate else parse.quote_plus(validate)}',
+            f'seccode={parse.quote_plus(seccode)}',
+            f'challenge={parse.quote_plus(challenge)}',
+            f'validate={parse.quote_plus(validate)}',
 
         ]
         params = self.user.sort_and_sign(extra_params)
-        print(params)
 
-        url = 'https://passport.bilibili.com/api/v3/oauth2/login'
+        url = f'https://passport.bilibili.com/api/v3/oauth2/login?{params}'
 
-        json_rsp = self.__web_session.request_json(
-            'POST', url, headers=self.user.dict_bili['appheaders'], params=params)
-        print(json_rsp)
+        json_rsp = await self._web_session.request_json(
+            'POST', url, headers=self.user.dict_bili['appheaders'], params=None)
+        print(f'login response: {json_rsp}')
         if json_rsp['code'] == -105:
             url = json_rsp['data']['url']
             return url
-        raise Exception('get_captcha', json_rsp)
+        return None
 
 
 class CrackClient:
-    def __init__(self, url):
-        self.__web_session = WebSession()
-        self.__url = url
+    def __init__(self, url, web_session: WebSession):
+        self._web_session = web_session
+        self._url = url
 
-    def request_crack(self, url):
+    async def request_crack(self, url: str) -> Any:
         data = {
             'url': url,
         }
-        json_rsp = self.__web_session.request_json('POST', f'{self.__url}/crack', json=data)
+        json_rsp = await self._web_session.request_json('POST', f'{self._url}/crack', json=data, keep_try=False)
         return json_rsp
 
 
-def main():
-    bili = Bili()
-    crack_client = CrackClient('http://127.0.0.1:3333')
+async def one_try():
+    web_session = WebSession()
+    bili = Bili(web_session)
+    crack_client = CrackClient('http://127.0.0.1:3333', web_session)
+    args = await bili.get_key()
     while True:
-        args = bili.get_key()
-        url = bili.login(*args, '', '', '')
-        print(f'url {url}')
-        result = crack_client.request_crack(url=url)
-        print(f'Result: {result}')
-        print('=' * 100, need_timestamp=False)
-        if not result['code']:
-            data = result['data']
-            bili.login(*args, challenge=data['challenge'], seccode=data['seccode'], validate=data['validate'])
+        url = await bili.login(*args, '', '', '')
+        if url is not None:
+            break
+    print(f'url {url}')
+    result = await crack_client.request_crack(url=url)
+    print(f'request_crack result: {result}')
+    if not result['code']:
+        data = result['data']
+        await bili.login(*args, challenge=data['challenge'], seccode=data['seccode'], validate=data['validate'])
+    await web_session.session.close()
+
 
-        time.sleep(5)
+async def main():
+    start_time = time.time()
+    await asyncio.gather(*[one_try() for _ in range(4)])
+    print("--- %s seconds ---" % (time.time() - start_time))
 
 
 if __name__ == "__main__":
-    main()
+    asyncio.get_event_loop().run_until_complete(main())
diff --git a/request_websession.py b/request_websession.py
diff --git a/requirements.txt b/requirements.txt
@@ -1,8 +1,9 @@
 requests==2.25.1
 toml==0.10.2
-selenium==3.141.0
 scikit-image==0.18.1
 opencv_python==4.5.1.48
 numpy==1.20.1
-flask==1.1.2
-rsa==4.7.2
+rsa==4.7.2
+playwright==1.9.1
+aiohttp==3.7.4post0
+aiojobs==0.3.0
diff --git a/server.py b/server.py
@@ -1,40 +1,55 @@
-from flask import Flask, jsonify, request
+import asyncio
+from typing import Tuple
+
+from aiohttp import web
+from aiojobs.aiohttp import atomic, setup
+from playwright.async_api import async_playwright, BrowserContext
 
 from server_core.cracker import Cracker
 from server_core.track_maker import TrackMaker
 from server_core.find_gap_position import check_gap_position
+from aiohttp_websession import WebSession
+
 
-track_maker = TrackMaker()
+async def init() -> Tuple[BrowserContext, WebSession, TrackMaker]:
+    return await (await (await async_playwright().start()).chromium.launch(headless=True)).new_context(viewport={'width': 350, 'height': 560}), WebSession(), TrackMaker()
 
-app = Flask(__name__)
 
+context, session, track_maker = asyncio.get_event_loop().run_until_complete(init())
 
-@app.route("/")
-def root():
-    jsonify({
+
+@atomic
+async def root(_: web.Request):
+    return web.json_response({
         'code': 0,
         'data': {
-            'msg': 'HELLO WORLD.'
-        },
-    })
+            'msg': 'Hello world.'
+        }})
 
 
-@app.route("/crack", methods=['POST'])
-def crack():
-    data = request.get_json()
-    print(data['url'])
+@atomic
+async def crack(request: web.Request):
+    try:
+        json_data = await request.json()
+    except:
+        return web.json_response({
+            'code': -1,
+            'data': {
+                'msg': 'Hello world.'
+            }})
 
-    cracker = Cracker('C:/Program Files (x86)/Google/Chrome/Application/chromedriver.exe')
+    page = await context.new_page()
+    cracker = Cracker(page, session)
 
-    load_result = cracker.load_url(url=data['url'])
+    load_result = await cracker.load_url(url=json_data['url'])
     while load_result:
-        cracker.refresh()
-        load_result = cracker.load_url()
+        await cracker.refresh()
+        load_result = await cracker.load_url()
 
-    reordered_fullbg_img, reordered_bg_img, gap_img = cracker.fetch_imgs()
-    gap_position = check_gap_position(reordered_fullbg_img, reordered_bg_img, gap_img, verbose=True)
+    reordered_fullbg_img, reordered_bg_img, gap_img = await cracker.fetch_imgs()
+    gap_position = check_gap_position(reordered_fullbg_img, reordered_bg_img, gap_img, verbose=False)
 
-    ratio = cracker.position2actual_distance(reordered_bg_img)
+    ratio = await cracker.position2actual_distance(reordered_bg_img)
 
     # resized_reordered_bg_img = cv2.resize(reordered_bg_img, None, fx=ratio, fy=ratio, interpolation=cv2.INTER_AREA)
     # cv2.line(resized_reordered_bg_img,
@@ -49,11 +64,20 @@ def crack():
     print('生成的轨迹', distance, ratio, distance * ratio, track)
 
     # cracker.slide_slider(track, ratio)
-    cracker.test_slide_slider(distance, ratio)
-    geetest_seccode, geetest_challenge, geetest_validate = cracker.get_result()
+    await cracker.test_slide_slider(distance, ratio)
+    geetest_seccode, geetest_challenge, geetest_validate = await cracker.get_result()
+
+    print({
+        'seccode': geetest_seccode,
+        'challenge': geetest_challenge,
+        'validate': geetest_validate,
+    })
+    print(json_data['url'])
+
+    await page.close()
 
-    if geetest_challenge is not None:
-        return jsonify({
+    if geetest_challenge:
+        return web.json_response({
             'code': 0,
             'data': {
                 'seccode': geetest_seccode,
@@ -62,11 +86,15 @@ def crack():
             },
         })
 
-    return jsonify({
+    return web.json_response({
         'code': -1,
         'data': {},
     })
 
 
 if __name__ == "__main__":
-    app.run('0.0.0.0', '3333')
+    app = web.Application()
+    setup(app)
+    app.router.add_route('GET', '/', root)
+    app.router.add_route('POST', '/crack', crack)
+    web.run_app(app, port=3333)
diff --git a/server_core/cracker.py b/server_core/cracker.py
