118 lines
3.7 KiB
Python
118 lines
3.7 KiB
Python
"""Парсинг HTML phpBB: темы и посты.
|
|
|
|
Структура phpBB (subsilver2, нива-лада.рф):
|
|
|
|
Список тем в разделе:
|
|
<a href="./viewtopic.php?f=22&t=123" class="topictitle">...</a>
|
|
|
|
Посты в теме (табличная вёрстка subsilver2):
|
|
<tr>
|
|
<td class="profile" rowspan="2">
|
|
<strong><a href="./memberlist.php?...">AuthorName</a></strong><br />
|
|
<span class="postdetails">Role</span>
|
|
...
|
|
</td>
|
|
<td>
|
|
<div class="postbody">текст поста</div>
|
|
...
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td>
|
|
<div class="gensmall" style="float: left;">
|
|
<a href="./memberlist.php?...">AuthorName</a> » Дата
|
|
</div>
|
|
</td>
|
|
</tr>
|
|
"""
|
|
|
|
import re, logging
|
|
from bs4 import BeautifulSoup
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def parse_topics(soup: BeautifulSoup) -> list:
|
|
"""Список тем со страницы раздела."""
|
|
items = []
|
|
|
|
for a in soup.select("a.topictitle"):
|
|
href = a.get("href", "")
|
|
m = re.search(r"[?&]t=(\d+)", href)
|
|
if not m:
|
|
continue
|
|
topic_id = int(m.group(1))
|
|
f_match = re.search(r"[?&]f=(\d+)", href)
|
|
forum_id = int(f_match.group(1)) if f_match else 0
|
|
items.append({
|
|
"id": topic_id,
|
|
"forum_id": forum_id,
|
|
"title": a.text.strip(),
|
|
"url": href if href.startswith("http") else None,
|
|
})
|
|
|
|
return items
|
|
|
|
|
|
def parse_posts(soup: BeautifulSoup) -> list:
|
|
"""Посты со страницы темы (subsilver2, нива-лада.рф).
|
|
|
|
Структура одного поста (3 <tr> в <table class="tablebg">):
|
|
|
|
<tr class="row1"> ← шапка: автор + дата
|
|
<td><b class="postauthor">Author</b></td>
|
|
<td>
|
|
<div style="float: right;">
|
|
<b>Добавлено:</b> 12 авг 2025, 20:00
|
|
</div>
|
|
</td>
|
|
</tr>
|
|
<tr class="row1"> ← тело: аватар + текст
|
|
<td class="profile">...</td>
|
|
<td><div class="postbody">текст</div></td>
|
|
</tr>
|
|
<tr class="row1"> ← подвал
|
|
<td class="profile">Вернуться к началу</td>
|
|
<td><div class="gensmall">...</div></td>
|
|
</tr>
|
|
"""
|
|
out = []
|
|
|
|
# Ищем все блоки: b.postauthor = автор, затем ищем postbody
|
|
for author_b in soup.select("b.postauthor"):
|
|
author = author_b.text.strip()
|
|
|
|
# Дата: ищем "Добавлено:" в том же tr
|
|
date = ""
|
|
tr_header = author_b.find_parent("tr")
|
|
if tr_header:
|
|
date_td = tr_header.select_one("td[width='100%']")
|
|
if date_td:
|
|
dt_text = date_td.get_text(" ", strip=True)
|
|
m = re.search(r"Добавлено:\s*(.+)", dt_text)
|
|
if m:
|
|
date = m.group(1).strip()
|
|
|
|
# Текст: ищем div.postbody в следующем tr после tr_header
|
|
text = ""
|
|
if tr_header:
|
|
tr_body = tr_header.find_next_sibling("tr")
|
|
if tr_body:
|
|
pb = tr_body.select_one("div.postbody")
|
|
if pb:
|
|
for h in pb.select("div[style*='none'], span[style*='none'], "
|
|
"blockquote cite, dl.codebox, div.codebox"):
|
|
h.decompose()
|
|
text = pb.get_text("\n", strip=True)
|
|
|
|
num = str(len(out) + 1)
|
|
|
|
out.append({
|
|
"author": author,
|
|
"date": date,
|
|
"num": f"#{num}",
|
|
"text": text,
|
|
})
|
|
|
|
return out
|