eriks/erepublik-client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable HTTP2 to overcome CloudFlare captchas

Browse Source
This commit is contained in:
KEriks 2022-03-21 14:55:07 +02:00
parent c86a5ffc5a
commit c9dbf7ee1e
2 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
erepublik/access_points.py
View File

@ -4,8 +4,7 @@ import random
import time import time
from typing import Any, Dict, List, Mapping, Union from typing import Any, Dict, List, Mapping, Union
from requests import Response, Session from httpx import Response, Client as Session, RequestError
from requests.exceptions import ConnectionError
from requests_toolbelt.utils import dump from requests_toolbelt.utils import dump
from erepublik import constants, utils from erepublik import constants, utils
@ -15,11 +14,12 @@ __all__ = ["SlowRequests", "CitizenAPI"]
class SlowRequests(Session): class SlowRequests(Session):
last_time: datetime.datetime last_time: datetime.datetime
timeout: datetime.timedelta = datetime.timedelta(milliseconds=500) __timeout: datetime.timedelta = datetime.timedelta(milliseconds=500)
debug: bool = False debug: bool = False
http2 = True
def __init__(self, proxies: Dict[str, str] = None, user_agent: str = None): def __init__(self, proxies: Dict[str, str] = None, user_agent: str = None):
super().__init__() super().__init__(http2=True, follow_redirects=True)
if proxies: if proxies:
self.proxies = proxies self.proxies = proxies
if user_agent is None: if user_agent is None:
@ -27,13 +27,13 @@ class SlowRequests(Session):
self.request_log_name = utils.get_file(utils.now().strftime("debug/requests_%Y-%m-%d.log")) self.request_log_name = utils.get_file(utils.now().strftime("debug/requests_%Y-%m-%d.log"))
self.last_time = utils.now() self.last_time = utils.now()
self.headers.update({"User-Agent": user_agent}) self.headers.update({"User-Agent": user_agent})
self.hooks["response"] = [self._log_response] self.event_hooks["response"] = [self._log_response]
@property @property
def as_dict(self): def as_dict(self):
return dict( return dict(
last_time=self.last_time, last_time=self.last_time,
timeout=self.timeout, __timeout=self.__timeout,
cookies=self.cookies.get_dict(), cookies=self.cookies.get_dict(),
debug=self.debug, debug=self.debug,
user_agent=self.headers["User-Agent"], user_agent=self.headers["User-Agent"],
@ -46,14 +46,14 @@ class SlowRequests(Session):
self._log_request(url, method, **kwargs) self._log_request(url, method, **kwargs)
try: try:
resp = super().request(method, url, *args, **kwargs) resp = super().request(method, url, *args, **kwargs)
except ConnectionError: except RequestError:
time.sleep(1) time.sleep(1)
return self.request(method, url, *args, **kwargs) return self.request(method, url, *args, **kwargs)
# self._log_response(resp) # self._log_response(resp)
return resp return resp
def _slow_down_requests(self): def _slow_down_requests(self):
ltt = utils.good_timedelta(self.last_time, self.timeout) ltt = utils.good_timedelta(self.last_time, self.__timeout)
if ltt > utils.now(): if ltt > utils.now():
seconds = (ltt - utils.now()).total_seconds() seconds = (ltt - utils.now()).total_seconds()
time.sleep(seconds if seconds > 0 else 0) time.sleep(seconds if seconds > 0 else 0)
@ -81,7 +81,7 @@ class SlowRequests(Session):
def _log_response(self, response: Response, *args, **kwargs): def _log_response(self, response: Response, *args, **kwargs):
redirect = kwargs.get("redirect") redirect = kwargs.get("redirect")
url = response.request.url url = str(response.request.url)
if self.debug: if self.debug:
if response.history and not redirect: if response.history and not redirect:
for hist_resp in response.history: for hist_resp in response.history:
@ -92,7 +92,7 @@ class SlowRequests(Session):
fd_time = self.last_time.strftime("%Y/%m/%d/%H-%M-%S") fd_time = self.last_time.strftime("%Y/%m/%d/%H-%M-%S")
fd_name = utils.slugify(url[len(CitizenBaseAPI.url) :]) fd_name = utils.slugify(url[len(CitizenBaseAPI.url) :])
fd_extra = "_REDIRECT" if redirect else "" fd_extra = "_REDIRECT" if redirect else ""
response.read()
try: try:
utils.json.loads(response.text) utils.json.loads(response.text)
fd_ext = "json" fd_ext = "json"
@ -102,9 +102,9 @@ class SlowRequests(Session):
filename = f"{fd_path}/{fd_time}_{fd_name}{fd_extra}.{fd_ext}" filename = f"{fd_path}/{fd_time}_{fd_name}{fd_extra}.{fd_ext}"
utils.write_file(filename, response.text) utils.write_file(filename, response.text)
if not redirect: #if not redirect:
data = dump.dump_all(response) # data = dump.dump_all(response)
utils.write_file(f"debug/dumps/{fd_time}_{fd_name}{fd_extra}.{fd_ext}.dump", data.decode("utf8")) # utils.write_file(f"debug/dumps/{fd_time}_{fd_name}{fd_extra}.{fd_ext}.dump", data.decode("utf8"))
@staticmethod @staticmethod
def get_random_user_agent() -> str: def get_random_user_agent() -> str:
@ -145,7 +145,7 @@ class CitizenBaseAPI:
return dict(url=self.url, request=self._req.as_dict, token=self.token) return dict(url=self.url, request=self._req.as_dict, token=self.token)
def post(self, url: str, data=None, json=None, **kwargs) -> Response: def post(self, url: str, data=None, json=None, **kwargs) -> Response:
return self._req.post(url, data, json, **kwargs) return self._req.post(url, data=data, json=json, **kwargs)
def get(self, url: str, **kwargs) -> Response: def get(self, url: str, **kwargs) -> Response:
return self._req.get(url, **kwargs) return self._req.get(url, **kwargs)

1
requirements.txt
View File

@ -1,4 +1,5 @@
PySocks==1.7.1 PySocks==1.7.1
pytz>2021.0 pytz>2021.0
httpx==0.22.0
requests>2.25.0,!=2.27.0 requests>2.25.0,!=2.27.0
requests-toolbelt==0.9.1 requests-toolbelt==0.9.1