clean old db and network code

requirements.txt

@@ -1,17 +1,12 @@
 playwright>=1.33.0
-taos
 better-exceptions>=0.2.2
-DBUtils>=2.0
 parsel>=1.5.2
 requests>=2.22.0
 selenium>=3.141.0
 bs4>=0.0.1
-ipython>=7.14.0
 bitarray>=1.5.3
-cryptography>=3.3.2
 urllib3>=1.25.8
 loguru>=0.5.3
-pyperclip>=1.8.2
 webdriver-manager>=3.5.3
 terminal-layout>=2.1.3
 playwright_stealth #==1.0.5

setup.py

@@ -18,21 +18,12 @@
 
 requires = [
     "better-exceptions>=0.2.2",
-    "DBUtils>=2.0",
     "parsel>=1.5.2",
-    "PyMySQL>=0.9.3",
-    "redis>=2.10.6,<4.0.0",
     "requests>=2.22.0",
     "bs4>=0.0.1",
-    "ipython>=7.14.0",
-    "redis-py-cluster>=2.1.0",
-    "cryptography>=3.3.2",
     "selenium>=3.141.0",
-    "pymongo>=3.10.1",
     "urllib3>=1.25.8",
     "loguru>=0.5.3",
-    "influxdb>=5.3.1",
-    "pyperclip>=1.8.2",
     "webdriver-manager>=3.5.3",
     "terminal-layout>=2.1.3",
     "playwright",

tsup/__init__.py

@@ -30,7 +30,4 @@
 # from tsup.login import *
 # from tsup.db import *
 # from tsup.tiktok import *
-from tsup.utils.custom_argparse import ArgumentParser
-from tsup.network.request import Request
-from tsup.network.response import Response
-from tsup.network.item import Item, UpdateItem
+from tsup.utils.custom_argparse import ArgumentParser

tsup/utils/tools.py

@@ -41,14 +41,12 @@
 from urllib import request
 from urllib.parse import urljoin
 
-import redis
 import requests
 import six
 from requests.cookies import RequestsCookieJar
 from w3lib.url import canonicalize_url as _canonicalize_url
 
 import tsup.setting as setting
-from tsup.db.redisdb import RedisDB
 from tsup.utils.email_sender import EmailSender
 from tsup.utils.log import log
 
@@ -2730,38 +2728,6 @@ def re_def_supper_class(obj, supper_class):
 freq_limit_record = {}
 
 
-def reach_freq_limit(rate_limit, *key):
-    """
-    频率限制
-    :param rate_limit: 限制时间 单位秒
-    :param key: 频率限制的key
-    :return: True / False
-    """
-    if rate_limit == 0:
-        return False
-
-    msg_md5 = get_md5(*key)
-    key = "rate_limit:{}".format(msg_md5)
-    try:
-        if get_redisdb().strget(key):
-            return True
-
-        get_redisdb().strset(key, time.time(), ex=rate_limit)
-    except redis.exceptions.ConnectionError as e:
-        # 使用内存做频率限制
-        global freq_limit_record
-
-        if key not in freq_limit_record:
-            freq_limit_record[key] = time.time()
-            return False
-
-        if time.time() - freq_limit_record.get(key) < rate_limit:
-            return True
-        else:
-            freq_limit_record[key] = time.time()
-
-    return False
-
 
 def dingding_warning(
     message, message_prefix=None, rate_limit=None, url=None, user_phone=None

tsup/utils/webdriver/playwright_driver_async.py

@@ -25,7 +25,6 @@
 from playwright.async_api import Response
 from playwright.async_api import async_playwright
 
-from tsup.utils import tools
 from tsup.utils.log import log
 from tsup.utils.webdriver.webdirver import *
 import filecmp
@@ -289,9 +288,7 @@ async def quit(self):
         if self.driver:
             await self.driver.stop()
 
-    @property
-    def domain(self):
-        return tools.get_domain(self.url or self.page.url)
+
 
     @property
     def cookies(self):

tsup/utils/webdriver/playwright_driver_sync.py

@@ -25,7 +25,6 @@
 from playwright.sync_api import Response
 from playwright.sync_api import sync_playwright
 
-from tsup.utils import tools
 from tsup.utils.log import log
 from tsup.utils.webdriver.webdirver import *
 
@@ -178,9 +177,7 @@ def quit(self):
         self.browser.close()
         self.driver.stop()
 
-    @property
-    def domain(self):
-        return tools.get_domain(self.url or self.page.url)
+
 
     @property
     def cookies(self):