Learn Python 3 Spider

https://github.com/wistbean/learn_python3_spider
Admin
A comprehensive Python web scraping tutorial series covering everything from basic scraping...
Tokens:49,606
Snippets:65
Trust Score:7.2
License:MIT
Update:6 months ago
Show doc for...
Tokens:
Raw
### Selenium WebDriver Initialization

Source: https://context7.com/wistbean/learn_python3_spider/llms.txt

This snippet shows the basic setup for using Selenium with ChromeDriver to automate browser actions. It initializes the Chrome driver, sets a wait time for elements, and configures the browser window size. Ensure you have Selenium installed and ChromeDriver executable in your system's PATH or specified path.

```python
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
import xlwt

# 初始化浏览器
browser = webdriver.Chrome()
WAIT = WebDriverWait(browser, 10)
browser.set_window_size(1400, 900)

```

--------------------------------

### Multi-threaded Web Scraping

Source: https://github.com/wistbean/learn_python3_spider/blob/master/README.md

Python example demonstrating multi-threaded web scraping. This approach is useful for speeding up the process of fetching data from multiple URLs.

```Python
import requests
import threading
from queue import Queue

# Target website for scraping (example: a site with many images)
TARGET_URL = 'http://example.com/images'
NUM_THREADS = 5

# Queue to hold URLs to be processed
url_queue = Queue()
# List to store results (e.g., image URLs or content)
results = []

def worker():
    while True:
        url = url_queue.get()
        if url is None: # Sentinel value to stop the thread
            break
        try:
            response = requests.get(url, timeout=10)
            response.raise_for_status()
            # Process the response content here
            # For example, extract image links or save content
            # This is a placeholder for actual scraping logic
            content_length = len(response.content)
            results.append({'url': url, 'content_length': content_length})
            print(f"Processed {url}, Content Length: {content_length}")
        except requests.exceptions.RequestException as e:
            print(f"Error processing {url}: {e}")
        finally:
            url_queue.task_done()

def main():
    # Populate the queue with URLs to scrape
    # In a real scenario, you'd generate these URLs based on the website structure
    for i in range(1, 21): # Example: scrape 20 pages
        url_queue.put(f"{TARGET_URL}?page={i}")

    threads = []
    for _ in range(NUM_THREADS):
        t = threading.Thread(target=worker)
        t.start()
        threads.append(t)

    # Wait for all tasks in the queue to be processed
    url_queue.join()

    # Stop worker threads by adding None sentinel values
    for _ in range(NUM_THREADS):
        url_queue.put(None)
    for t in threads:
        t.join()

    print("\n--- Scraping Complete ---")
    print(f"Total items processed: {len(results)}")
    # Optionally print some results
    # for res in results[:5]:
    #     print(res)

if __name__ == '__main__':
    main()

```

--------------------------------

### Multi-threaded Image Download with concurrent.futures

Source: https://context7.com/wistbean/learn_python3_spider/llms.txt

This snippet demonstrates how to download images concurrently using Python's concurrent.futures module and threading. It includes functions for building request headers, fetching web pages, parsing URLs, and downloading images to a specified directory. Ensure necessary libraries like requests, beautifulsoup4, and lxml are installed.

```python
import concurrent
import os
from concurrent.futures import ProcessPoolExecutor
import requests
from bs4 import BeautifulSoup


def header(referer):
    """构建带有Referer的请求头（防盗链）"""
    return {
        'Host': 'i.meizitu.net',
        'Pragma': 'no-cache',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
        'Cache-Control': 'no-cache',
        'Connection': 'keep-alive',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36',
        'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
        'Referer': '{}'.format(referer),
    }


def request_page(url):
    """请求页面"""
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
    except requests.RequestException:
        return None


def get_page_urls():
    """获取所有页面的链接"""
    urls = []
    for i in range(1, 2):
        baseurl = 'https://www.mzitu.com/page/{}'.format(i)
        html = request_page(baseurl)
        soup = BeautifulSoup(html, 'lxml')
        elements = soup.find(class_='postlist').find_all('li')

        for item in elements:
            url = item.find('span').find('a').get('href')
            print('页面链接：%s' % url)
            urls.append(url)
    return urls


def download_Pic(title, image_list):
    """下载图片到指定目录"""
    os.mkdir(title)
    j = 1
    for item in image_list:
        filename = '%s/%s.jpg' % (title, str(j))
        print('downloading....%s : NO.%s' % (title, str(j)))
        with open(filename, 'wb') as f:
            img = requests.get(item, headers=header(item)).content
            f.write(img)
        j += 1


def download(url):
    """解析页面并下载所有图片"""
    html = request_page(url)
    soup = BeautifulSoup(html, 'lxml')
    total = soup.find(class_='pagenavi').find_all('a')[-2].find('span').string
    title = soup.find('h2').string
    image_list = []

    for i in range(int(total)):
        html = request_page(url + '/%s' % (i + 1))
        soup = BeautifulSoup(html, 'lxml')
        img_url = soup.find('img').get('src')
        image_list.append(img_url)

    download_Pic(title, image_list)


def download_all_images(list_page_urls):
    """使用进程池并发下载"""
    with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor:
        for url in list_page_urls:
            executor.submit(download, url)


if __name__ == '__main__':
    list_page_urls = get_page_urls()
    download_all_images(list_page_urls)

# 输出示例:
# 页面链接：https://www.mzitu.com/123456
# downloading....美女图集 : NO.1
# downloading....美女图集 : NO.2

```

--------------------------------

### Initialize Excel Workbook

Source: https://context7.com/wistbean/learn_python3_spider/llms.txt

Sets up the initial Excel file structure with headers for video data.

```python
book = xlwt.Workbook(encoding='utf-8', style_compression=0)
sheet = book.add_sheet('蔡徐坤篮球', cell_overwrite_ok=True)
sheet.write(0, 0, '名称')
sheet.write(0, 1, '地址')
sheet.write(0, 2, '描述')
sheet.write(0, 3, '观看次数')
sheet.write(0, 4, '弹幕数')
sheet.write(0, 5, '发布时间')

n = 1
```

--------------------------------

### Initialize and Render Bar Chart with pyecharts

Source: https://github.com/wistbean/learn_python3_spider/blob/master/GaoKao_Score/2006-2017山西高考录取分数线.html

Initializes a pyecharts chart instance and sets its options to render a bar chart. Requires an HTML element with the specified ID and the pyecharts library to be included.

```javascript
var chart_5320919801744e71a61d5784b3299df3 = echarts.init( document.getElementById('5320919801744e71a61d5784b3299df3'), 'white', {renderer: 'canvas'}); var option_5320919801744e71a61d5784b3299df3 = { "color": [ "#c23531", "#2f4554", "#61a0a8", "#d48265", "#749f83", "#ca8622", "#bda29a", "#6e7074", "#546570", "#c4ccd3", "#f05b72", "#ef5b9c", "#f47920", "#905a3d", "#fab27b", "#2a5caa", "#444693", "#726930", "#b2d235", "#6d8346", "#ac6767", "#1d953f", "#6950a1", "#918597" ], "series": [ { "type": "bar", "name": "\u6587\u79d1(\u4e00\u672c)", "data": [ 518, 518, 513, 526, 507, 539, 543, 533, 548, 545, 580, 561 ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } }, { "type": "bar", "name": "\u7406\u79d1(\u4e00\u672c)", "data": [ 481, 519, 515, 534, 493, 530, 570, 536, 547, 546, 572, 552 ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } }, { "type": "bar", "name": "\u6587\u79d1(\u4e8c\u672c)", "data": [ "\n\t\t\t\t\t452\n\t\t\t\t", "\n\t\t\t\t\t460\n\t\t\t\t", "\n\t\t\t\t\t462\n\t\t\t\t", "\n\t\t\t\t\t478\n\t\t\t\t", "\n\t\t\t\t\t459\n\t\t\t\t", "\n\t\t\t\t\t492\n\t\t\t\t", "\n\t\t\t\t\t496\n\t\t\t\t", "\n\t\t\t\t\t492\n\t\t\t\t", "\n\t\t\t\t\t507\n\t\t\t\t", "\n\t\t\t\t\t510\n\t\t\t\t", "\n\t\t\t\t\t553\n\t\t\t\t", "\n\t\t\t\t\t528\n\t\t\t\t" ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } }, { "type": "bar", "name": "\u7406\u79d1(\u4e8c\u672c)", "data": [ "\n\t\t\t\t\t400\n\t\t\t\t", "\n\t\t\t\t\t438\n\t\t\t\t", "\n\t\t\t\t\t442\n\t\t\t\t", "\n\t\t\t\t\t462\u00a0\n\t\t\t\t", "\n\t\t\t\t\t440\u00a0\n\t\t\t\t", "\n\t\t\t\t\t476\n\t\t\t\t", "\n\t\t\t\t\t520\n\t\t\t\t", "\n\t\t\t\t\t491\n\t\t\t\t", "\n\t\t\t\t\t505\n\t\t\t\t", "\n\t\t\t\t\t511\n\t\t\t\t", "\n\t\t\t\t\t439\n\t\t\t\t", "\n\t\t\t\t\t515\n\t\t\t\t" ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } } ], "legend": [ { "data": [ "\u6587\u79d1(\u4e00\u672c)", "\u7406\u79d1(\u4e00\u672c)", "\u6587\u79d1(\u4e8c\u672c)", "\u7406\u79d1(\u4e8c\u672c)" ], "selected": { "\u6587\u79d1(\u4e00\u672c)": true, "\u7406\u79d1(\u4e00\u672c)": true, "\u6587\u79d1(\u4e8c\u672c)": true, "\u7406\u79d1(\u4e8c\u672c)": true }, "show": true } ], "tooltip": { "show": true, "trigger": "item", "triggerOn": "mousemove|click", "axisPointer": { "type": "line" }, "textStyle": { "fontSize": 14 }, "borderWidth": 0 }, "xAxis": [ { "show": true, "scale": false, "nameLocation": "end", "nameGap": 15, "gridIndex": 0, "inverse": false, "offset": 0, "splitNumber": 5, "minInterval": 0, "splitLine": { "show": false, "lineStyle": { "width": 1, "opacity": 1, "curveness": 0, "type": "solid" } }, "data": [ "\n2017\n", "\n2016\n", "\n2015\n", "\n2014\n", "\n2013\n", "\n2012\n", "\n2011\n", "\n2010\n", "\n2009\n", "\n2008\n", "\n2007\n", "\n2006\n" ] } ], "yAxis": [ { "show": true, "scale": false, "nameLocation": "end", "nameGap": 15, "gridIndex": 0, "inverse": false, "offset": 0, "splitNumber": 5, "minInterval": 0, "splitLine": { "show": false, "lineStyle": { "width": 1, "opacity": 1, "curveness": 0, "type": "solid" } } } ], "title": [ { "text": "\n2006-2017\u5c71\u897f\u9ad8\u8003\u5f55\u53d6\u5206\u6570\u636e\u7ebf\uff08\u6c47\u603b\uff09\n" } ] }; chart_5320919801744e71a61d5784b3299df3.setOption(option_5320919801744e71a61d5784b3299df3);
```

--------------------------------

### Scrape Emojis

Source: https://github.com/wistbean/learn_python3_spider/blob/master/README.md

Python script to scrape a large collection of emojis. This example focuses on extracting image URLs or data for emojis.

```Python
import requests
from bs4 import BeautifulSoup
import os

# URL of the website containing emojis (example: a site listing emojis)
# Replace with an actual URL if you have one in mind.
EMOJI_URL = 'http://example.com/emojis' 
DOWNLOAD_DIR = 'downloaded_emojis'

def download_emojis(url):
    if not os.path.exists(DOWNLOAD_DIR):
        os.makedirs(DOWNLOAD_DIR)

    try:
        response = requests.get(url, timeout=10)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, 'html.parser')

        # Find emoji elements. This selector is hypothetical and depends on the website's structure.
        # Look for <img> tags or elements containing emoji data.
        emoji_elements = soup.find_all('img', class_='emoji') # Example selector

        if not emoji_elements:
            print("No emoji elements found with the specified selector.")
            # Try another common pattern: elements with data-src or similar attributes
            emoji_elements = soup.select('div[data-emoji-url]') # Another hypothetical selector
            if not emoji_elements:
                print("Tried alternative selector, still no emojis found.")
                return

        print(f"Found {len(emoji_elements)} potential emoji elements.")

        for i, element in enumerate(emoji_elements):
            emoji_url = None
            # Try to get the image URL from common attributes
            if element.name == 'img':
                emoji_url = element.get('src') or element.get('data-src')
            elif element.get('data-emoji-url'): # For custom attributes
                emoji_url = element.get('data-emoji-url')

            if emoji_url:
                # Ensure the URL is absolute
                if not emoji_url.startswith('http'):
                    emoji_url = requests.compat.urljoin(url, emoji_url)
                
                try:
                    print(f"Downloading emoji {i+1}: {emoji_url}")
                    emoji_response = requests.get(emoji_url, stream=True, timeout=10)
                    emoji_response.raise_for_status()

                    # Determine filename
                    filename = os.path.basename(emoji_url)
                    if not filename: # If URL has no filename, generate one
                        filename = f"emoji_{i+1}.png" # Assume png, adjust if needed
                    
                    filepath = os.path.join(DOWNLOAD_DIR, filename)
                    
                    with open(filepath, 'wb') as f:
                        for chunk in emoji_response.iter_content(chunk_size=8192):
                            f.write(chunk)
                    print(f"Saved: {filepath}")

                except requests.exceptions.RequestException as e:
                    print(f"Error downloading {emoji_url}: {e}")
                except IOError as e:
                    print(f"Error saving file {filename}: {e}")
            else:
                print(f"Could not extract URL for emoji element {i+1}.")

    except requests.exceptions.RequestException as e:
        print(f"Error fetching URL {url}: {e}")
    except Exception as e:
        print(f"An unexpected error occurred: {e}")

if __name__ == '__main__':
    print(f"Attempting to download emojis from {EMOJI_URL} into '{DOWNLOAD_DIR}' directory.")
    print("Note: This script uses a placeholder URL and selectors. You need to adapt it.")
    # download_emojis(EMOJI_URL)
    print("Download function is commented out. Uncomment and update EMOJI_URL and selectors to run.")

```

--------------------------------

### Create Bar Chart with Pyecharts

Source: https://github.com/wistbean/learn_python3_spider/blob/master/GaoKao_Score/2006-2017重庆高考录取分数线.html

Initializes a pyecharts chart and sets its options to render a bar chart. Ensure the target DOM element exists and pyecharts library is included.

```javascript
var chart_826d7f0f17b947828e227285a0f99adf = echarts.init( document.getElementById('826d7f0f17b947828e227285a0f99adf'), 'white', {renderer: 'canvas'}); var option_826d7f0f17b947828e227285a0f99adf = { "color": [ "#c23531", "#2f4554", "#61a0a8", "#d48265", "#749f83", "#ca8622", "#bda29a", "#6e7074", "#546570", "#c4ccd3", "#f05b72", "#ef5b9c", "#f47920", "#905a3d", "#fab27b", "#2a5caa", "#444693", "#726930", "#b2d235", "#6d8346", "#ac6767", "#1d953f", "#6950a1", "#918597" ], "series": [ { "type": "bar", "name": "\u6587\u79d1(\u4e00\u672c)", "data": [ 525, 527, 572, 555, 556, 554, 564, 573, 546, 576, 540, 556 ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } }, { "type": "bar", "name": "\u7406\u79d1(\u4e00\u672c)", "data": [ 492, 525, 573, 514, 520, 522, 533, 533, 557, 544, 505, 531 ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } }, { "type": "bar", "name": "\u6587\u79d1(\u4e8c\u672c)", "data": [ "\n\t\t\t\t\t436\n\t\t\t\t", "\n\t\t\t\t\t435\n\t\t\t\t", "\n\t\t\t\t\t532\n\t\t\t\t", "\n\t\t\t\t\t507\n\t\t\t\t", "\n\t\t\t\t\t499\n\t\t\t\t", "\n\t\t\t\t\t494\n\t\t\t\t", "\n\t\t\t\t\t504\n\t\t\t\t", "\n\t\t\t\t\t510\n\t\t\t\t", "\n\t\t\t\t\t480\n\t\t\t\t", "\n\t\t\t\t\t510\n\t\t\t\t", "\n\t\t\t\t\t476\n\t\t\t\t", "\n\t\t\t\t\t486\n\t\t\t\t" ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } }, { "type": "bar", "name": "\u7406\u79d1(\u4e8c\u672c)", "data": [ "\n\t\t\t\t\t395\n\t\t\t\t", "\n\t\t\t\t\t416\n\t\t\t\t", "\n\t\t\t\t\t527\n\t\t\t\t", "\n\t\t\t\t\t455\n\t\t\t\t", "\n\t\t\t\t\t462\n\t\t\t\t", "\n\t\t\t\t\t462\n\t\t\t\t", "\n\t\t\t\t\t479\n\t\t\t\t", "\n\t\t\t\t\t478\n\t\t\t\t", "\n\t\t\t\t\t502\n\t\t\t\t", "\n\t\t\t\t\t488\n\t\t\t\t", "\n\t\t\t\t\t450\n\t\t\t\t", "\n\t\t\t\t\t476\n\t\t\t\t" ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } } ], "legend": [ { "data": [ "\u6587\u79d1(\u4e00\u672c)", "\u7406\u79d1(\u4e00\u672c)", "\u6587\u79d1(\u4e8c\u672c)", "\u7406\u79d1(\u4e8c\u672c)" ], "selected": { "\u6587\u79d1(\u4e00\u672c)": true, "\u7406\u79d1(\u4e00\u672c)": true, "\u6587\u79d1(\u4e8c\u672c)": true, "\u7406\u79d1(\u4e8c\u672c)": true }, "show": true } ], "tooltip": { "show": true, "trigger": "item", "triggerOn": "mousemove|click", "axisPointer": { "type": "line" }, "textStyle": { "fontSize": 14 }, "borderWidth": 0 }, "xAxis": [ { "show": true, "scale": false, "nameLocation": "end", "nameGap": 15, "gridIndex": 0, "inverse": false, "offset": 0, "splitNumber": 5, "minInterval": 0, "splitLine": { "show": false, "lineStyle": { "width": 1, "opacity": 1, "curveness": 0, "type": "solid" } }, "data": [ "\n2017\n", "\n2016\n", "\n2015\n", "\n2014\n", "\n2013\n", "\n2012\n", "\n2011\n", "\n2010\n", "\n2009\n", "\n2008\n", "\n2007\n", "\n2006\n" ] } ], "yAxis": [ { "show": true, "scale": false, "nameLocation": "end", "nameGap": 15, "gridIndex": 0, "inverse": false, "offset": 0, "splitNumber": 5, "minInterval": 0, "splitLine": { "show": false, "lineStyle": { "width": 1, "opacity": 1, "curveness": 0, "type": "solid" } } } ], "title": [ { "text": "\n2006-2017\u91cd\u5e86\u9ad8\u8003\u5f55\u53d6\u5206\u6570\u636e\u7ebf\uff08\u6c47\u603b\uff09\n" } ] }; chart_826d7f0f17b947828e227285a0f99adf.setOption(option_826d7f0f17b947828e227285a0f99adf);
```

--------------------------------

### Search and Navigation Functions

Source: https://context7.com/wistbean/learn_python3_spider/llms.txt

Handles the initial search query and pagination logic using Selenium WebDriver.

```python
def search():
    """执行搜索操作"""
    try:
        print('开始访问b站....')
        browser.get("https://www.bilibili.com/")

        # 定位搜索框和按钮
        input = WAIT.until(EC.presence_of_element_located((By.CSS_SELECTOR, "#nav_searchform > input")))
        submit = WAIT.until(EC.element_to_be_clickable(
            (By.XPATH, '/html/body/div[2]/div/div[1]/div[1]/div/div[2]/div/form/div/button')))

        input.send_keys('蔡徐坤 篮球')
        submit.click()

        # 切换到新窗口
        print('跳转到新窗口')
        all_h = browser.window_handles
        browser.switch_to.window(all_h[1])
        get_source()

        # 获取总页数
        total = WAIT.until(EC.presence_of_element_located((By.CSS_SELECTOR,
            "#all-list > div.flow-loader > div.page-wrap > div > ul > li.page-item.last > button")))
        return int(total.text)
    except TimeoutException:
        return search()


def next_page(page_num):
    """翻到下一页"""
    try:
        print('获取下一页数据')
        next_btn = WAIT.until(EC.element_to_be_clickable((By.CSS_SELECTOR,
            '#all-list > div.flow-loader > div.page-wrap > div > ul > li.page-item.next > button')))
        next_btn.click()
        WAIT.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR,
            '#all-list > div.flow-loader > div.page-wrap > div > ul > li.page-item.active > button'),
            str(page_num)))
        get_source()
    except TimeoutException:
        browser.refresh()
        return next_page(page_num)
```

--------------------------------

### Crawl Dangdang Top 500 Books

Source: https://github.com/wistbean/learn_python3_spider/blob/master/README.md

Source code for crawling the top 500 highly-rated books from Dangdang.com. This example demonstrates basic web scraping techniques.

```Python
import requests
from bs4 import BeautifulSoup

def get_html(url):
    try:
        response = requests.get(url, timeout=10)
        response.raise_for_status()  # Raise an exception for bad status codes
        return response.text
    except requests.exceptions.RequestException as e:
        print(f"Error fetching URL {url}: {e}")
        return None

def parse_html(html):
    if not html:
        return []
    soup = BeautifulSoup(html, 'html.parser')
    books = []
    # Example: Find book elements (this would be specific to Dangdang's HTML structure)
    # For demonstration, let's assume we are looking for elements with class 'book-item'
    for item in soup.find_all('div', class_='book-item'):
        title_element = item.find('div', class_='title')
        author_element = item.find('div', class_='author')
        price_element = item.find('div', class_='price')
        
        title = title_element.text.strip() if title_element else 'N/A'
        author = author_element.text.strip() if author_element else 'N/A'
        price = price_element.text.strip() if price_element else 'N/A'
        
        books.append({
            'title': title,
            'author': author,
            'price': price
        })
    return books

def main():
    url = 'http://category.dangdang.com/pg1-cat100000.html' # Example URL, actual URL might differ
    html = get_html(url)
    books = parse_html(html)
    
    if books:
        print(f"Found {len(books)} books:")
        for book in books:
            print(book)
    else:
        print("No books found or an error occurred.")

if __name__ == '__main__':
    main()

```

--------------------------------

### Scrapy for Qiushibaike

Source: https://github.com/wistbean/learn_python3_spider/blob/master/README.md

Python code using the Scrapy framework to crawl jokes from Qiushibaike.com and store them in MongoDB. This example demonstrates setting up a Scrapy project and item pipelines.

```Python
# This is a conceptual representation of a Scrapy spider and item pipeline.
# A full Scrapy project would include multiple files (e.g., items.py, settings.py, pipelines.py).

import scrapy
from scrapy.exceptions import DropItem
from pymongo import MongoClient

# --- items.py (Conceptual)
class JokeItem(scrapy.Item):
    content = scrapy.Field()
    author = scrapy.Field()
    upvotes = scrapy.Field()

# --- spiders/qiushibaike_spider.py (Conceptual)
class QiushibaikeSpider(scrapy.Spider):
    name = 'qiushibaike'
    start_urls = ['https://www.qiushibaike.com/text/page/1/']

    def parse(self, response):
        for joke_div in response.css('div.q-item'):
            item = JokeItem()
            item['content'] = joke_div.css('div.content::text').get().strip()
            item['author'] = joke_div.css('div.author a::text').get().strip() if joke_div.css('div.author a::text').get() else 'Anonymous'
            item['upvotes'] = joke_div.css('span.stats-vote i::text').get().strip() if joke_div.css('span.stats-vote i::text').get() else '0'
            yield item

        # Follow pagination link
        next_page = response.css('li.next a::attr(href)').get()
        if next_page is not None:
            yield response.follow(next_page, self.parse)

# --- pipelines.py (Conceptual)
class MongoDBPipeline:
    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
        )

    def open_spider(self, spider):
        self.client = MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        if isinstance(item, JokeItem):
            # Check if joke already exists to avoid duplicates (optional)
            if self.db.jokes.find_one({'content': item['content']}):
                raise DropItem(f"Duplicate joke found: {item['content'][:50]}...")
            else:
                self.db.jokes.insert_one(dict(item))
                print(f"Inserted joke from {item['author']} into MongoDB.")
                return item
        else:
            return item

# --- settings.py (Conceptual - relevant parts)
# ITEM_PIPELINES = {
#     'your_project_name.pipelines.MongoDBPipeline': 300,
# }
# MONGO_URI = 'mongodb://localhost:27017/'
# MONGO_DATABASE = 'qiushibaike_db'

# To run this, you would set up a Scrapy project and place these conceptual parts
# into the correct files (items.py, spiders/your_spider_name.py, pipelines.py).
# Then run: scrapy crawl qiushibaike

```

--------------------------------

### Crawl Douban Top 250 Movies

Source: https://github.com/wistbean/learn_python3_spider/blob/master/README.md

Python script to scrape the top 250 most popular movies from Douban. This example involves parsing HTML and extracting specific data points.

```Python
import requests
from bs4 import BeautifulSoup
import time

def get_html(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
    }
    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        return response.text
    except requests.exceptions.RequestException as e:
        print(f"Error fetching URL {url}: {e}")
        return None

def parse_movie_page(html):
    if not html:
        return []
    soup = BeautifulSoup(html, 'html.parser')
    movies = []
    for movie_item in soup.find_all('div', class_='item'):
        title = movie_item.find('span', class_='title').text.strip() if movie_item.find('span', class_='title') else 'N/A'
        rating = movie_item.find('span', class_='rating_num').text.strip() if movie_item.find('span', class_='rating_num') else 'N/A'
        quote = movie_item.find('span', class_='inq').text.strip() if movie_item.find('span', class_='inq') else 'N/A'
        
        movies.append({
            'title': title,
            'rating': rating,
            'quote': quote
        })
    return movies

def main():
    base_url = 'https://movie.douban.com/top250'
    all_movies = []
    for i in range(10):  # Douban Top 250 has 10 pages
        url = f"{base_url}?start={i*25}&filter="
        print(f"Crawling page: {url}")
        html = get_html(url)
        movies_on_page = parse_movie_page(html)
        all_movies.extend(movies_on_page)
        time.sleep(1) # Be polite and avoid overwhelming the server

    print(f"Total movies crawled: {len(all_movies)}")
    for movie in all_movies:
        print(movie)

if __name__ == '__main__':
    main()

```

--------------------------------

### Create a Bar Chart with Pyecharts

Source: https://github.com/wistbean/learn_python3_spider/blob/master/GaoKao_Score/2006-2018青海高考录取分数线.html

Initializes a bar chart using pyecharts and sets its options. Requires an HTML element with the specified ID to render the chart.

```javascript
var chart_2964b1b8b4c84b5b8bef65431c71d688 = echarts.init( document.getElementById('2964b1b8b4c84b5b8bef65431c71d688'), 'white', {renderer: 'canvas'}); var option_2964b1b8b4c84b5b8bef65431c71d688 = { "color": [ "#c23531", "#2f4554", "#61a0a8", "#d48265", "#749f83", "#ca8622", "#bda29a", "#6e7074", "#546570", "#c4ccd3", "#f05b72", "#ef5b9c", "#f47920", "#905a3d", "#fab27b", "#2a5caa", "#444693", "#726930", "#b2d235", "#6d8346", "#ac6767", "#1d953f", "#6950a1", "#918597" ], "series": [ { "type": "bar", "name": "\u6587\u79d1(\u4e00\u672c)", "data": [ 475, 463, 457, 466, 473, 435, 433, 430, 430, 443, 490, 495, 452 ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } }, { "type": "bar", "name": "\u7406\u79d1(\u4e00\u672c)", "data": [ 403, 391, 416, 400, 406, 383, 401, 380, 405, 400, 438, 467, 434 ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } }, { "type": "bar", "name": "\u6587\u79d1(\u4e8c\u672c)", "data": [ "\n\t\t\t\t\t424\n\t\t\t\t", "\n\t\t\t\t\t421\n\t\t\t\t", "\n\t\t\t\t\t415\n\t\t\t\t", "\n\t\t\t\t\t420\n\t\t\t\t", "\n\t\t\t\t\t426\n\t\t\t\t", "\n\t\t\t\t\t382\n\t\t\t\t", "\n\t\t\t\t\t372\n\t\t\t\t", "\n\t\t\t\t\t380\n\t\t\t\t", "\n\t\t\t\t\t386\n\t\t\t\t", "\n\t\t\t\t\t405\n\t\t\t\t", "\n\t\t\t\t\t425\n\t\t\t\t", "\n\t\t\t\t\t418\n\t\t\t\t", "\n\t\t\t\t\t386\n\t\t\t\t" ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } }, { "type": "bar", "name": "\u7406\u79d1(\u4e8c\u672c)", "data": [ "\n\t\t\t\t\t365\n\t\t\t\t", "\n\t\t\t\t\t356\n\t\t\t\t", "\n\t\t\t\t\t380\n\t\t\t\t", "\n\t\t\t\t\t363\n\t\t\t\t", "\n\t\t\t\t\t362\n\t\t\t\t", "\n\t\t\t\t\t340\n\t\t\t\t", "\n\t\t\t\t\t355\n\t\t\t\t", "\n\t\t\t\t\t331\n\t\t\t\t", "\n\t\t\t\t\t363\n\t\t\t\t", "\n\t\t\t\t\t365\n\t\t\t\t", "\n\t\t\t\t\t388\n\t\t\t\t", "\n\t\t\t\t\t405\n\t\t\t\t", "\n\t\t\t\t\t374\n\t\t\t\t" ], "barCategoryGap": "20%", "label": { "show": true, "position": "top", "margin": 8 } } ], "legend": [ { "data": [ "\u6587\u79d1(\u4e00\u672c)", "\u7406\u79d1(\u4e00\u672c)", "\u6587\u79d1(\u4e8c\u672c)", "\u7406\u79d1(\u4e8c\u672c)" ], "selected": { "\u6587\u79d1(\u4e00\u672c)": true, "\u7406\u79d1(\u4e00\u672c)": true, "\u6587\u79d1(\u4e8c\u672c)": true, "\u7406\u79d1(\u4e8c\u672c)": true }, "show": true } ], "tooltip": { "show": true, "trigger": "item", "triggerOn": "mousemove|click", "axisPointer": { "type": "line" }, "textStyle": { "fontSize": 14 }, "borderWidth": 0 }, "xAxis": [ { "show": true, "scale": false, "nameLocation": "end", "nameGap": 15, "gridIndex": 0, "inverse": false, "offset": 0, "splitNumber": 5, "minInterval": 0, "splitLine": { "show": false, "lineStyle": { "width": 1, "opacity": 1, "curveness": 0, "type": "solid" } }, "data": [ "\n\t\t\t\t\t2018\n", "\n2017\n", "\n2016\n", "\n2015\n", "\n2014\n", "\n2013\n", "\n2012\n", "\n2011\n", "\n2010\n", "\n2009\n", "\n2008\n", "\n2007\n", "\n2006\n" ] } ], "yAxis": [ { "show": true, "scale": false, "nameLocation": "end", "nameGap": 15, "gridIndex": 0, "inverse": false, "offset": 0, "splitNumber": 5, "minInterval": 0, "splitLine": { "show": false, "lineStyle": { "width": 1, "opacity": 1, "curveness": 0, "type": "solid" } } } ], "title": [ { "text": "\n2006-2018\u9752\u6d77\u9ad8\u8003\u5f55\u53d6\u5206\u6570\u636e\u7ebf\uff08\u6c47\u603b\uff09\n" } ] }; chart_2964b1b8b4c84b5b8bef65431c71d688.setOption(option_2964b1b8b4c84b5b8bef65431c71d688);
```

--------------------------------

### Crawl Bilibili NBA Content

Source: https://github.com/wistbean/learn_python3_spider/blob/master/README.md

Python script to scrape content related to NBA and specific personalities from Bilibili. This example might involve handling dynamic content or specific API calls.

```Python
import requests
import json

def get_bilibili_videos(keyword):
    videos = []
    # This is a simplified example. Actual Bilibili API calls can be more complex.
    # You might need to find the correct API endpoints and parameters.
    search_url = f"https://api.bilibili.com/x/web-interface/search/type?keyword={keyword}&search_type=video&page=1"
    
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    }

    try:
        response = requests.get(search_url, headers=headers, timeout=10)
        response.raise_for_status()
        data = response.json()
        
        if data and data['code'] == 0 and 'data' in data and 'result' in data['data']:
            for item in data['data']['result']:
                videos.append({
                    'title': item.get('title', 'N/A'),
                    'aid': item.get('aid', 'N/A'),
                    'play': item.get('play', 'N/A'),
                    'bvid': item.get('bvid', 'N/A')
                })
        else:
            print(f"API response error or no results found for keyword: {keyword}")
            print(f"Response: {data}")

    except requests.exceptions.RequestException as e:
        print(f"Error fetching Bilibili data for keyword {keyword}: {e}")
    except json.JSONDecodeError:
        print(f"Error decoding JSON response for keyword: {keyword}")
        
    return videos

def main():
    keyword = "蔡徐坤"
    print(f"Searching Bilibili for videos related to: {keyword}")
    videos = get_bilibili_videos(keyword)
    
    if videos:
        print(f"Found {len(videos)} videos:")
        for video in videos:
            print(video)
    else:
        print("No videos found or an error occurred.")

if __name__ == '__main__':
    main()

```

--------------------------------

### Main Execution Block

Source: https://context7.com/wistbean/learn_python3_spider/llms.txt

The main entry point for the script, orchestrating the WeChat automation process from initialization to data scraping.

```python
if __name__ == '__main__':
    wc_moment = Wechat_Moment()
    wc_moment.login()
    wc_moment.find_xiaoshuaib()
    wc_moment.get_data()
```
Tokens:
Raw