From 6cfa0ff8554e562f3cbccd52984873a7c7b3026e Mon Sep 17 00:00:00 2001 From: QIN2DIM <62018067+QIN2DIM@users.noreply.github.com> Date: Fri, 1 Sep 2023 11:35:32 +0800 Subject: [PATCH] Perf(claim): change to async style (#205) * Perf(claim): change to async style --- Dockerfile | 4 +- requirements.txt | 2 +- src/claim.py | 26 +++--- src/services/agents/epic_games.py | 133 ++++++++++++++---------------- src/services/models.py | 62 +++++++++++++- 5 files changed, 140 insertions(+), 87 deletions(-) diff --git a/Dockerfile b/Dockerfile index cff2600..8a93f3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM python:3.10 as builder +WORKDIR /home/epic/src + COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt @@ -8,6 +10,4 @@ RUN apt update -y \ && playwright install firefox \ && playwright install-deps firefox -WORKDIR /home/epic/src - COPY src ./ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0a18338..ed88e0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ beautifulsoup4>=4.10.0 lxml>=4.9.3 # [extra_requires.solver] -hcaptcha-challenger[playwright]==0.7.0 +hcaptcha-challenger[playwright]==0.7.5 # [extra_requires.notify] apprise==1.1.0 diff --git a/src/claim.py b/src/claim.py index b4ed7eb..cd464cc 100644 --- a/src/claim.py +++ b/src/claim.py @@ -3,16 +3,17 @@ # Author : QIN2DIM # GitHub : https://github.com/QIN2DIM # Description: +import asyncio import sys import hcaptcha_challenger as solver from loguru import logger -from playwright.sync_api import BrowserContext +from playwright.async_api import BrowserContext from services.agents.epic_games import EpicPlayer, EpicGames from services.agents.epic_games import get_promotions, get_order_history -solver.install(upgrade=True, flush_yolo=True) +solver.install(flush_yolo=True) player = EpicPlayer.from_account() @@ -21,6 +22,7 @@ promotions = [] ctx_cookies_is_available = None +@logger.catch def prelude(): global promotions, ctx_cookies_is_available @@ -34,7 +36,10 @@ def prelude(): # Create tasks orders = get_order_history(player.cookies) namespaces = [order.namespace for order in orders] - promotions = [p for p in get_promotions() if p.namespace not in namespaces] + pros = get_promotions() + for pro in pros: + logger.debug("prelude", action="check", title=pro.title, url=pro.url) + promotions = [p for p in pros if p.namespace not in namespaces] if not promotions: logger.success( @@ -45,7 +50,7 @@ def prelude(): sys.exit() -def claim_epic_games(context: BrowserContext): +async def claim_epic_games(context: BrowserContext): global promotions page = context.pages[0] @@ -54,8 +59,8 @@ def claim_epic_games(context: BrowserContext): # Authorize if not ctx_cookies_is_available: logger.info("claim_epic_games", action="Try to flush cookie") - if epic.authorize(page): - epic.flush_token(context) + if await epic.authorize(page): + await epic.flush_token(context) else: logger.error( "claim_epic_games", action="Exit test case", reason="Failed to flush token" @@ -73,17 +78,16 @@ def claim_epic_games(context: BrowserContext): return # Execute - epic.claim_weekly_games(page, promotions) + await epic.claim_weekly_games(page, promotions) -@logger.catch -def run(): +async def run(): prelude() # Cookie is unavailable or need to process promotions agent = player.build_agent() - agent.execute(sequence=[claim_epic_games], headless=False) + await agent.execute(sequence=[claim_epic_games], headless=True) if __name__ == "__main__": - run() + asyncio.run(run()) diff --git a/src/services/agents/epic_games.py b/src/services/agents/epic_games.py index 20ad9f1..666a5e6 100644 --- a/src/services/agents/epic_games.py +++ b/src/services/agents/epic_games.py @@ -15,8 +15,7 @@ from typing import List, Dict import httpx from hcaptcha_challenger.agents.playwright.control import AgentT from loguru import logger -from playwright.sync_api import BrowserContext, expect, TimeoutError -from playwright.sync_api import Page +from playwright.async_api import BrowserContext, expect, TimeoutError, Page from services.models import EpicPlayer from utils import from_dict_to_model @@ -88,62 +87,68 @@ class EpicGames: self._promotions = self._promotions or get_promotions() return self._promotions - def _login(self, page: Page) -> str | None: - page.goto(URL_CLAIM, wait_until="domcontentloaded") - while page.locator('a[role="button"]:has-text("Sign In")').count() > 0: - page.goto(URL_LOGIN, wait_until="domcontentloaded") + async def _login(self, page: Page) -> str | None: + await page.goto(URL_CLAIM, wait_until="domcontentloaded") + while await page.locator('a[role="button"]:has-text("Sign In")').count() > 0: + await page.goto(URL_LOGIN, wait_until="domcontentloaded") logger.info("login", url=page.url) - page.click("#login-with-epic") + await page.click("#login-with-epic") logger.info("login-with-epic", url=page.url) - page.fill("#email", self.player.email) - page.type("#password", self.player.password) - page.click("#sign-in") + await page.fill("#email", self.player.email) + await page.type("#password", self.player.password) + await page.click("#sign-in") fall_in_challenge = False for _ in range(15): if not fall_in_challenge: with suppress(TimeoutError): - page.wait_for_url(URL_CLAIM, timeout=10000) + await page.wait_for_url(URL_CART_SUCCESS, timeout=3000) break - if not self._solver.qr: - return + logger.debug("claim_weekly_games", action="handle challenge") fall_in_challenge = True - result = self._solver(window="login", recur_url=URL_CLAIM) - if result in [ - self._solver.status.CHALLENGE_BACKCALL, - self._solver.status.CHALLENGE_RETRY, - ]: - page.click("//a[@class='talon_close_button']") - page.wait_for_timeout(1000) - page.click("#sign-in", delay=200) - continue - if result == self._solver.status.CHALLENGE_SUCCESS: - page.wait_for_url(URL_CLAIM) - break + result = await self._solver(window="login", recur_url=URL_CLAIM) + logger.debug("handle challenge", result=result) + match result: + case self._solver.status.CHALLENGE_BACKCALL: + await page.click("//a[@class='talon_close_button']") + await page.wait_for_timeout(1000) + await page.click("#sign-in", delay=200) + case self._solver.status.CHALLENGE_RETRY: + continue + # await page.reload() + # await page.fill("#email", self.player.email) + # await page.type("#password", self.player.password) + # await page.click("#sign-in") + case self._solver.status.CHALLENGE_SUCCESS: + with suppress(TimeoutError): + await page.wait_for_url(URL_CLAIM) + break + return - return self._solver.status.AUTH_SUCCESS + return self._solver.status.CHALLENGE_SUCCESS - def authorize(self, page: Page): + async def authorize(self, page: Page): for _ in range(3): - result = self._login(page) - if result not in [self._solver.status.CHALLENGE_SUCCESS]: - continue - return True + match await self._login(page): + case self._solver.status.CHALLENGE_SUCCESS: + return True + case _: + continue logger.critical("Failed to flush token", agent=self.__class__.__name__) - def flush_token(self, context: BrowserContext): + async def flush_token(self, context: BrowserContext): page = context.pages[0] - page.goto("https://www.epicgames.com/account/personal", wait_until="networkidle") - page.goto( + await page.goto("https://www.epicgames.com/account/personal", wait_until="networkidle") + await page.goto( "https://store.epicgames.com/zh-CN/p/orwell-keeping-an-eye-on-you", wait_until="networkidle", ) - context.storage_state(path=self.player.ctx_cookie_path) + await context.storage_state(path=self.player.ctx_cookie_path) self.player.ctx_cookies.reload(self.player.ctx_cookie_path) logger.success("flush_token", path=self.player.ctx_cookie_path) - def claim_weekly_games(self, page: Page, promotions: List[Game]): + async def claim_weekly_games(self, page: Page, promotions: List[Game]): """ :param page: @@ -153,69 +158,57 @@ class EpicGames: # --> Add promotions to Cart for promotion in promotions: logger.info("claim_weekly_games", action="go to store", url=promotion.url) - page.goto(promotion.url, wait_until="load") + await page.goto(promotion.url, wait_until="load") # <-- Handle pre-page with suppress(TimeoutError): - page.click("//button//span[text()='Continue']", timeout=3000) + await page.click("//button//span[text()='Continue']", timeout=3000) # --> Make sure promotion is not in the library before executing cta_btn = page.locator("//aside//button[@data-testid='add-to-cart-cta-button']") - text = cta_btn.text_content() + text = await cta_btn.text_content() if text == "View In Cart": continue if text == "Add To Cart": - cta_btn.click() - expect(cta_btn).to_have_text("View In Cart") + await cta_btn.click() + await expect(cta_btn).to_have_text("View In Cart") # --> Goto cart page - page.goto(URL_CART, wait_until="domcontentloaded") - page.click("//button//span[text()='Check Out']") + await page.goto(URL_CART, wait_until="domcontentloaded") + await page.click("//button//span[text()='Check Out']") # <-- Handle Any LICENSE with suppress(TimeoutError): - page.click("//label[@for='agree']", timeout=2000) + await page.click("//label[@for='agree']", timeout=2000) accept = page.locator("//button//span[text()='Accept']") - if accept.is_enabled(): - accept.click() + if await accept.is_enabled(): + await accept.click() # --> Move to webPurchaseContainer iframe logger.info("claim_weekly_games", action="move to webPurchaseContainer iframe") wpc = page.frame_locator("//iframe[@class='']") payment_btn = wpc.locator("//div[@class='payment-order-confirm']") with suppress(Exception): - expect(payment_btn).to_be_attached() - page.wait_for_timeout(2000) - payment_btn.click() + await expect(payment_btn).to_be_attached() + await page.wait_for_timeout(2000) + await payment_btn.click() logger.info("claim_weekly_games", action="click payment button") # <-- Insert challenge - fall_in_challenge = False - for _ in range(15): - if not fall_in_challenge: - with suppress(TimeoutError): - page.wait_for_url(URL_CART_SUCCESS, timeout=10000) + # {{< if fall in challenge >}} + match await self._solver(window="free", recur_url=URL_CART_SUCCESS): + case self._solver.status.CHALLENGE_BACKCALL | self._solver.status.CHALLENGE_RETRY: + await wpc.locator("//a[@class='talon_close_button']").click() + await page.wait_for_timeout(1000) + await payment_btn.click(delay=200) + case self._solver.status.CHALLENGE_SUCCESS: + await page.wait_for_url(URL_CART_SUCCESS) break - logger.debug("claim_weekly_games", action="handle challenge") - fall_in_challenge = True - result = self._solver(window="free", recur_url=URL_CART_SUCCESS) - logger.debug("claim_weekly_games", action="challenge", result=result) - if result in [ - self._solver.status.CHALLENGE_BACKCALL, - self._solver.status.CHALLENGE_RETRY, - ]: - wpc.locator("//a[@class='talon_close_button']").click() - page.wait_for_timeout(1000) - payment_btn.click(delay=200) - continue - if result == self._solver.status.CHALLENGE_SUCCESS: - page.wait_for_url(URL_CART_SUCCESS) - break # --> Wait for success - page.wait_for_url(URL_CART_SUCCESS) + await page.wait_for_url(URL_CART_SUCCESS) logger.success("claim_weekly_games", action="success", url=page.url) diff --git a/src/services/models.py b/src/services/models.py index e43de8a..df84742 100644 --- a/src/services/models.py +++ b/src/services/models.py @@ -6,17 +6,22 @@ from __future__ import annotations import abc +import inspect import json import os import time from abc import ABC from contextlib import suppress from dataclasses import dataclass, field +from datetime import datetime, timedelta from pathlib import Path -from typing import Literal, Dict +from typing import Callable, Awaitable, List, Any, Dict +from typing import Literal import httpx -from hcaptcha_challenger.agents.playwright import Tarnished +from hcaptcha_challenger.agents.playwright.tarnished import Malenia +from loguru import logger +from playwright.async_api import async_playwright from settings import config, project @@ -60,6 +65,57 @@ class EpicCookie: pass +class Ring(Malenia): + @staticmethod + async def patch_cookies(context): + five_days_ago = datetime.now() - timedelta(days=5) + cookie = { + "name": "OptanonAlertBoxClosed", + "value": five_days_ago.isoformat(), + "domain": ".epicgames.com", + "path": "/", + } + await context.add_cookies([cookie]) + + async def execute( + self, + sequence: Callable[..., Awaitable[...]] | List, + *, + parameters: Dict[str, Any] = None, + headless: bool = False, + locale: str = "en-US", + **kwargs, + ): + async with async_playwright() as p: + context = await p.firefox.launch_persistent_context( + user_data_dir=self._user_data_dir, + headless=headless, + locale=locale, + record_video_dir=self._record_dir, + record_har_path=self._record_har_path, + args=["--hide-crash-restore-bubble"], + **kwargs, + ) + await self.apply_stealth(context) + await self.patch_cookies(context) + + if not isinstance(sequence, list): + sequence = [sequence] + for container in sequence: + logger.info("execute task", name=container.__name__) + kws = {} + params = inspect.signature(container).parameters + if parameters and isinstance(parameters, dict): + for name in params: + if name != "context" and name in parameters: + kws[name] = parameters[name] + if not kws: + await container(context) + else: + await container(context, **kws) + await context.close() + + @dataclass class Player(ABC): email: str @@ -127,7 +183,7 @@ class Player(ABC): return self.user_data_dir.joinpath("ctx_cookie.json") def build_agent(self): - return Tarnished( + return Ring( user_data_dir=self.browser_context_dir, record_dir=self.record_dir, record_har_path=self.record_har_path,