SimplePool/Test/test_spider.py

# -*- coding: utf-8 -*-
# @date：2023/12/29 8:58
# @Author：LiuYiJie
# @file： test_spider
"""爬虫模块，包含`SpiderMeta`类和一些初始的
爬虫类，如果用户需要定义自己的爬虫类，必须要继承
`SpiderMeta`类，并重写`gets`方法，`gets`
方法要求返回 ip:port 形式的代理。
"""

import time


class SpiderMeta(type):
    spiders = []

    def _init(cls):
        """子类的构造方法
        :return: None
        """
        cls._counter = 1

    def _increment(cls, count):
        """子类用于增加计数器的方法
        :param count: 计数器增加量
        :return: None
        """
        cls._counter += count

    def _flush(cls):
        """计数器刷新为 1
        :return: None
        """
        cls._counter = 1

    def __new__(cls, *args, **kwargs):
        """构造子类
        :param args: args[0] = name, args[1] = bases, args[2] = attrs.
        :param kwargs: No.
        :return: 新类
        """

        # 爬虫类必须要有 `get` 方法。
        if 'gets' not in args[2]:
            raise ValueError(args[0])

        # 给爬虫类添加一些默认方法
        args[2]['__init__'] = lambda self: SpiderMeta._init(self)
        args[2]['increment'] = lambda self, count: SpiderMeta._increment(self, count)
        args[2]['flush'] = lambda self: SpiderMeta._flush(self)

        # 将爬虫类加入到 `spiders` 列表中
        SpiderMeta.spiders.append(type.__new__(cls, *args, **kwargs))
        return type.__new__(cls, *args, **kwargs)


class Proxy360Spider(metaclass=SpiderMeta):
    start_url = 'http://www.proxy360.cn/default.aspx'

    def gets(self, page_total=None):
        ans = []
        soup = get_page(self.start_url)
        for proxy in soup.find_all('div', {'class': 'proxylistitem'}):
            item = proxy.find_all('span', {"class": "tbBottomLine"})
            ip = item[0].get_text().replace('\r\n', '').replace(' ', '')
            port = item[1].get_text().replace('\r\n', '').replace(' ', '')
            ans.append(':'.join([ip, port]))
        return ans


class Daili666Spider(metaclass=SpiderMeta):
    start_url = 'http://www.66ip.cn/{}.html'

    def gets(self, page_total=3):
        urls = [self.start_url.format(i)
                for i in range(self._counter, self._counter + page_total)]
        self.increment(page_total)
        ans = []
        for url in urls:
            soup = get_page(url)
            # 防止被 Ban, 加 1s 的间隔。
            time.sleep(1)
            proxy_list = soup.find('table', {"border": "2px"})
            for proxy in proxy_list.find_all('tr')[1:]:
                ip = proxy.find_all('td')[0].get_text()
                port = proxy.find_all('td')[1].get_text()
                ans.append(':'.join([ip, port]))
        return ans


class KuaidailiSpider(metaclass=SpiderMeta):
    start_url = 'http://www.kuaidaili.com/free/inha/{}/'

    def gets(self, page_total=2):
        urls = [self.start_url.format(i)
                for i in range(self._counter, self._counter + page_total)]
        self.increment(page_total)
        ans = []
        for url in urls:
            soup = get_page(url)
            time.sleep(1)
            proxy_list = soup.find('table',
                                   {'class': 'table table-bordered table-striped'}) \
                .find('tbody')
            for proxy in proxy_list.find_all('tr'):
                tmp = proxy.find_all('td')
                ip = tmp[0].get_text()
                port = tmp[1].get_text()
                ans.append(':'.join([ip, port]))
        return ans


class XiciSpider(metaclass=SpiderMeta):
    start_url = 'http://www.xicidaili.com/nn/{}'

    def gets(self, page_total=2):
        urls = [self.start_url.format(i)
                for i in range(self._counter, self._counter + page_total)]
        self.increment(page_total)
        ans = []
        for url in urls:
            soup = get_page(url)
            time.sleep(1)
            proxy_list = soup.find('table', {'id': 'ip_list'}) \
                             .find_all('tr')[1:]
            for proxy in proxy_list:
                tmp = proxy.find_all('td')
                ip = tmp[1].get_text()
                port = tmp[2].get_text()
                ans.append(':'.join([ip, port]))
        return ans


from fastapi import FastAPI, Query
from urllib.parse import quote
import uvicorn

app = FastAPI()

@app.get("/search/")
def search_items(query_param: str = Query(..., description="Your query parameter")):
    # 将查询参数进行 URL 编码
    encoded_query_param = quote(query_param)

    return {"result": f"Searching for {encoded_query_param}"}


if __name__ == '__main__':
    uvicorn.run('test_spider:app', host='0.0.0.0', port=8080)