refactor(core): replace JSON-based pipeline with MySQL database
This commit introduces a major architectural overhaul, migrating the data backend from a flat JSON file to a relational MySQL database. - Replaced multiple scraping scripts with a unified `main.py` that handles crawling, parsing, and database synchronization. - Introduced `mysql_helper.py` for robust database interaction with a connection pool. - Added `queries.sql` defining the new database schema for assets, categories, tags, and changelogs. - Removed all obsolete frontend code (v1, v2) and old scraping scripts. This change provides a more scalable and maintainable foundation for managing asset data.
This commit is contained in:
xiaomai
3
.gitignore
vendored
3
.gitignore
vendored
@@ -50,3 +50,6 @@ pnpm-debug.log*
|
|||||||
*.zip
|
*.zip
|
||||||
*.tar.gz
|
*.tar.gz
|
||||||
*.rar
|
*.rar
|
||||||
|
|
||||||
|
# Output
|
||||||
|
media
|
||||||
1
all_asset_infos.json
Normal file
1
all_asset_infos.json
Normal file
File diff suppressed because one or more lines are too long
1
all_asset_infos_detailed.json
Normal file
1
all_asset_infos_detailed.json
Normal file
File diff suppressed because one or more lines are too long
@@ -1,67 +0,0 @@
|
|||||||
<!DOCTYPE html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
||||||
<meta charset="UTF-8" />
|
|
||||||
<title>Kenney Asset Gallery</title>
|
|
||||||
<link
|
|
||||||
rel="stylesheet"
|
|
||||||
href="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/css/lightgallery.min.css"
|
|
||||||
integrity="sha512-QMCloGTsG2vNSnHcsxYTapI6pFQNnUP6yNizuLL5Wh3ha6AraI6HrJ3ABBaw6SIUHqlSTPQDs/SydiR98oTeaQ=="
|
|
||||||
crossorigin="anonymous"
|
|
||||||
referrerpolicy="no-referrer"
|
|
||||||
/>
|
|
||||||
<script
|
|
||||||
src="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/lightgallery.min.js"
|
|
||||||
integrity="sha512-n02TbYimj64qb98ed5WwkNiSw/i9Xlvv4Ehvhg0jLp3qMAMWCYUHbOMbppZ0vimtyiyw9NqNqxUZC4hq86f4aQ=="
|
|
||||||
crossorigin="anonymous"
|
|
||||||
referrerpolicy="no-referrer"
|
|
||||||
></script>
|
|
||||||
<link
|
|
||||||
rel="stylesheet"
|
|
||||||
href="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/css/lg-zoom.min.css"
|
|
||||||
integrity="sha512-S/hU6dGSK3D7SRpCvRF/IEufIr6Ikgp5vDiJarhdeFGEnw36hWZ6gVBjnwBbzjA+NEP7D8Gdm+5LL1HEsyiB1w=="
|
|
||||||
crossorigin="anonymous"
|
|
||||||
referrerpolicy="no-referrer"
|
|
||||||
/>
|
|
||||||
<script
|
|
||||||
src="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/plugins/zoom/lg-zoom.min.js"
|
|
||||||
integrity="sha512-fwxc/NvaA3du4ZRE6J/Ilrqi2xwOB1QfHBR4neA+ha13/pkweiRfPgBiV4VbfAf/Vi3rXAXdQ3zexUJ1V2bWrg=="
|
|
||||||
crossorigin="anonymous"
|
|
||||||
referrerpolicy="no-referrer"
|
|
||||||
></script>
|
|
||||||
<link
|
|
||||||
rel="stylesheet"
|
|
||||||
href="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/css/lg-thumbnail.min.css"
|
|
||||||
integrity="sha512-rKuOh3xlF/027KUPuMok0ESsZ2zWPRzkniD3n5zZKCAtbiVkYw66DR4KtVAGf8dLPLr5DdyQs05BlSmEyXctkQ=="
|
|
||||||
crossorigin="anonymous"
|
|
||||||
referrerpolicy="no-referrer"
|
|
||||||
/>
|
|
||||||
<script
|
|
||||||
src="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/plugins/thumbnail/lg-thumbnail.min.js"
|
|
||||||
integrity="sha512-jZxB8WysJ6S6e4Hz5IZpAzR1WiflBl0hBxriHGlLkUN32T18+rD1aLNifa1KTll/zx8lIfWVP1NqEjHi/Khy5w=="
|
|
||||||
crossorigin="anonymous"
|
|
||||||
referrerpolicy="no-referrer"
|
|
||||||
></script>
|
|
||||||
<link rel="stylesheet" href="style.css" />
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<h1>🎮 Kenney Asset Gallery</h1>
|
|
||||||
|
|
||||||
<!-- 筛选栏 -->
|
|
||||||
<div id="filters">
|
|
||||||
<label for="categoryFilter">选择分类：</label>
|
|
||||||
<select name="categoryFilter" id="categoryFilter">
|
|
||||||
<option value="all">📂 所有分类</option>
|
|
||||||
</select>
|
|
||||||
<label for="tagFilter">选择标签：</label>
|
|
||||||
<select name="tagFilter" id="tagFilter">
|
|
||||||
<option value="all">🏷️ 所有标签</option>
|
|
||||||
</select>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div id="gallery"></div>
|
|
||||||
|
|
||||||
<script src="script.js"></script>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@@ -1,98 +0,0 @@
|
|||||||
let allData = [];
|
|
||||||
|
|
||||||
function sanitize(str) {
|
|
||||||
return str.replaceAll("\\", "/");
|
|
||||||
}
|
|
||||||
|
|
||||||
function populateFilters(data) {
|
|
||||||
const catSet = new Set();
|
|
||||||
const tagSet = new Set();
|
|
||||||
|
|
||||||
data.forEach(item => {
|
|
||||||
catSet.add(item.properties?.Category?.[0]);
|
|
||||||
(item.properties?.Tags || []).forEach(tag => tagSet.add(tag));
|
|
||||||
});
|
|
||||||
|
|
||||||
const catFilter = document.getElementById("categoryFilter");
|
|
||||||
[...catSet].sort().forEach(cat => {
|
|
||||||
const option = document.createElement("option");
|
|
||||||
option.value = cat;
|
|
||||||
option.textContent = cat;
|
|
||||||
catFilter.appendChild(option);
|
|
||||||
});
|
|
||||||
|
|
||||||
const tagFilter = document.getElementById("tagFilter");
|
|
||||||
[...tagSet].sort().forEach(tag => {
|
|
||||||
const option = document.createElement("option");
|
|
||||||
option.value = tag;
|
|
||||||
option.textContent = tag;
|
|
||||||
tagFilter.appendChild(option);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function render(data) {
|
|
||||||
const gallery = document.getElementById("gallery");
|
|
||||||
gallery.innerHTML = ""; // clear
|
|
||||||
|
|
||||||
data.forEach((item, index) => {
|
|
||||||
const images = (item.images || []).map(sanitize);
|
|
||||||
const tags = (item.properties?.Tags || []).join(', ');
|
|
||||||
const category = item.properties?.Category?.[0] || 'Uncategorized';
|
|
||||||
const downloadPath = sanitize(item.download);
|
|
||||||
|
|
||||||
const card = document.createElement("div");
|
|
||||||
card.className = "card";
|
|
||||||
card.setAttribute("data-category", category);
|
|
||||||
card.setAttribute("data-tags", tags);
|
|
||||||
|
|
||||||
const galleryGroupId = `gallery-${index}`;
|
|
||||||
card.innerHTML = `
|
|
||||||
<div class="lg-gallery" id="${galleryGroupId}">
|
|
||||||
<a href="${images[0]}" data-lg-size="1400-800">
|
|
||||||
<img src="${images[0]}" alt="${item.title}">
|
|
||||||
</a>
|
|
||||||
${images.slice(1).map(img => `
|
|
||||||
<a href="${img}" data-lg-size="1400-800" style="display:none;"></a>
|
|
||||||
`).join("")}
|
|
||||||
</div>
|
|
||||||
<div class="card-body">
|
|
||||||
<div class="card-title">${item.title}</div>
|
|
||||||
<div class="card-tags">Tags: ${tags}</div>
|
|
||||||
<div class="card-footer">
|
|
||||||
<a class="download-btn" href="${downloadPath}" download>⬇️ 下载资源</a>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
`;
|
|
||||||
gallery.appendChild(card);
|
|
||||||
|
|
||||||
// 初始化 lightGallery
|
|
||||||
lightGallery(document.getElementById(galleryGroupId), {
|
|
||||||
selector: 'a',
|
|
||||||
thumbnail: true,
|
|
||||||
zoom: true
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function filterGallery() {
|
|
||||||
const cat = document.getElementById("categoryFilter").value;
|
|
||||||
const tag = document.getElementById("tagFilter").value;
|
|
||||||
|
|
||||||
const filtered = allData.filter(item => {
|
|
||||||
const matchCat = (cat === 'all') || (item.properties?.Category?.[0] === cat);
|
|
||||||
const matchTag = (tag === 'all') || (item.properties?.Tags || []).includes(tag);
|
|
||||||
return matchCat && matchTag;
|
|
||||||
});
|
|
||||||
|
|
||||||
render(filtered);
|
|
||||||
}
|
|
||||||
|
|
||||||
fetch("data/kenney_data_local.json")
|
|
||||||
.then(res => res.json())
|
|
||||||
.then(data => {
|
|
||||||
allData = data;
|
|
||||||
populateFilters(data);
|
|
||||||
render(data);
|
|
||||||
document.getElementById("categoryFilter").addEventListener("change", filterGallery);
|
|
||||||
document.getElementById("tagFilter").addEventListener("change", filterGallery);
|
|
||||||
});
|
|
||||||
@@ -1,82 +0,0 @@
|
|||||||
body {
|
|
||||||
font-family: "Segoe UI", sans-serif;
|
|
||||||
background-color: #f0f2f5;
|
|
||||||
margin: 0;
|
|
||||||
padding: 2rem;
|
|
||||||
color: #333;
|
|
||||||
}
|
|
||||||
|
|
||||||
h1 {
|
|
||||||
text-align: center;
|
|
||||||
margin-bottom: 2rem;
|
|
||||||
color: #444;
|
|
||||||
}
|
|
||||||
|
|
||||||
#gallery {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
|
|
||||||
gap: 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.card {
|
|
||||||
background: white;
|
|
||||||
border-radius: 12px;
|
|
||||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
|
|
||||||
overflow: hidden;
|
|
||||||
transition: transform 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.card:hover {
|
|
||||||
transform: translateY(-5px);
|
|
||||||
}
|
|
||||||
|
|
||||||
.card img {
|
|
||||||
width: 100%;
|
|
||||||
height: 200px;
|
|
||||||
object-fit: cover;
|
|
||||||
}
|
|
||||||
|
|
||||||
.card-body {
|
|
||||||
padding: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.card-title {
|
|
||||||
font-size: 1.2rem;
|
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.card-tags {
|
|
||||||
font-size: 0.85rem;
|
|
||||||
color: #666;
|
|
||||||
}
|
|
||||||
|
|
||||||
.card-footer {
|
|
||||||
margin-top: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.download-btn {
|
|
||||||
display: inline-block;
|
|
||||||
padding: 0.4rem 0.8rem;
|
|
||||||
background: #4caf50;
|
|
||||||
color: white;
|
|
||||||
border-radius: 6px;
|
|
||||||
text-decoration: none;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
transition: background 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.download-btn:hover {
|
|
||||||
background: #45a049;
|
|
||||||
}
|
|
||||||
|
|
||||||
#filters {
|
|
||||||
display: flex;
|
|
||||||
justify-content: center;
|
|
||||||
gap: 1rem;
|
|
||||||
margin-bottom: 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
padding: 0.5rem;
|
|
||||||
font-size: 1rem;
|
|
||||||
}
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
<!DOCTYPE html>
|
|
||||||
<html lang="zh-CN">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8" />
|
|
||||||
<title>Kenney 资源库</title>
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
||||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/css/lightgallery.min.css" integrity="sha512-QMCloGTsG2vNSnHcsxYTapI6pFQNnUP6yNizuLL5Wh3ha6AraI6HrJ3ABBaw6SIUHqlSTPQDs/SydiR98oTeaQ==" crossorigin="anonymous" referrerpolicy="no-referrer" />
|
|
||||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/lightgallery.min.js" integrity="sha512-n02TbYimj64qb98ed5WwkNiSw/i9Xlvv4Ehvhg0jLp3qMAMWCYUHbOMbppZ0vimtyiyw9NqNqxUZC4hq86f4aQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
|
|
||||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/css/lg-zoom.min.css" integrity="sha512-S/hU6dGSK3D7SRpCvRF/IEufIr6Ikgp5vDiJarhdeFGEnw36hWZ6gVBjnwBbzjA+NEP7D8Gdm+5LL1HEsyiB1w==" crossorigin="anonymous" referrerpolicy="no-referrer" />
|
|
||||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/plugins/zoom/lg-zoom.min.js" integrity="sha512-fwxc/NvaA3du4ZRE6J/Ilrqi2xwOB1QfHBR4neA+ha13/pkweiRfPgBiV4VbfAf/Vi3rXAXdQ3zexUJ1V2bWrg==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
|
|
||||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/css/lg-thumbnail.min.css" integrity="sha512-rKuOh3xlF/027KUPuMok0ESsZ2zWPRzkniD3n5zZKCAtbiVkYw66DR4KtVAGf8dLPLr5DdyQs05BlSmEyXctkQ==" crossorigin="anonymous" referrerpolicy="no-referrer" />
|
|
||||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/lightgallery/2.8.3/plugins/thumbnail/lg-thumbnail.min.js" integrity="sha512-jZxB8WysJ6S6e4Hz5IZpAzR1WiflBl0hBxriHGlLkUN32T18+rD1aLNifa1KTll/zx8lIfWVP1NqEjHi/Khy5w==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
|
|
||||||
<link rel="stylesheet" href="style.css" />
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<div class="container">
|
|
||||||
<aside class="sidebar">
|
|
||||||
<input type="text" id="searchInput" placeholder="🔍 搜索资源名称..." />
|
|
||||||
<ul id="resourceList"></ul>
|
|
||||||
</aside>
|
|
||||||
|
|
||||||
<main class="content">
|
|
||||||
<div id="details" class="hidden">
|
|
||||||
<h2 id="detailTitle"></h2>
|
|
||||||
<div id="detailTags"></div>
|
|
||||||
<div id="galleryWrapper"></div>
|
|
||||||
<div id="downloadBlock"></div>
|
|
||||||
<div id="versionTableWrapper"></div>
|
|
||||||
</div>
|
|
||||||
<div id="placeholder" class="placeholder">
|
|
||||||
← 请选择左侧资源以查看详情
|
|
||||||
</div>
|
|
||||||
</main>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<script src="libs/lightgallery.min.js"></script>
|
|
||||||
<script src="libs/lg-thumbnail.min.js"></script>
|
|
||||||
<script src="libs/lg-zoom.min.js"></script>
|
|
||||||
<script src="script.js"></script>
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
@@ -1,108 +0,0 @@
|
|||||||
let allData = [];
|
|
||||||
let currentActive = null;
|
|
||||||
|
|
||||||
// 定义一个函数，用于将字符串中的反斜杠替换为正斜杠
|
|
||||||
function sanitize(str) {
|
|
||||||
return str.replaceAll("\\", "/");
|
|
||||||
}
|
|
||||||
|
|
||||||
function renderList(data) {
|
|
||||||
const list = document.getElementById("resourceList");
|
|
||||||
list.innerHTML = "";
|
|
||||||
|
|
||||||
data.forEach((item, index) => {
|
|
||||||
const li = document.createElement("li");
|
|
||||||
li.setAttribute("data-index", index);
|
|
||||||
|
|
||||||
const thumb = item.images?.[0] ? sanitize(item.images[0]) : "";
|
|
||||||
const name = item.title;
|
|
||||||
const assets = item.properties?.Assets || "未知";
|
|
||||||
|
|
||||||
li.innerHTML = `
|
|
||||||
<img src="${thumb}" class="thumb" alt="">
|
|
||||||
<div>
|
|
||||||
<div><strong>${name}</strong></div>
|
|
||||||
<div style="font-size:0.85rem; color: #666;">素材量: ${assets}</div>
|
|
||||||
</div>
|
|
||||||
`;
|
|
||||||
|
|
||||||
li.addEventListener("click", () => showDetails(item, li));
|
|
||||||
list.appendChild(li);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function showDetails(item, li) {
|
|
||||||
if (currentActive) currentActive.classList.remove("active");
|
|
||||||
currentActive = li;
|
|
||||||
currentActive.classList.add("active");
|
|
||||||
|
|
||||||
document.getElementById("placeholder").classList.add("hidden");
|
|
||||||
document.getElementById("details").classList.remove("hidden");
|
|
||||||
|
|
||||||
document.getElementById("detailTitle").textContent = item.title;
|
|
||||||
document.getElementById("detailTags").textContent = `分类: ${
|
|
||||||
item.properties?.Category?.[0] || "N/A"
|
|
||||||
} | 标签: ${(item.properties?.Tags || []).join(", ")}`;
|
|
||||||
|
|
||||||
// 下载链接
|
|
||||||
const download = sanitize(item.download);
|
|
||||||
document.getElementById(
|
|
||||||
"downloadBlock"
|
|
||||||
).innerHTML = `<a class="download-btn" href="${download}" download>⬇️ 下载资源</a>`;
|
|
||||||
|
|
||||||
// 图集
|
|
||||||
const gallery = document.createElement("div");
|
|
||||||
gallery.id = "gallery";
|
|
||||||
item.images?.forEach((img) => {
|
|
||||||
img = sanitize(img);
|
|
||||||
const a = document.createElement("a");
|
|
||||||
a.href = img;
|
|
||||||
a.innerHTML = `<img src="${img}" alt="">`;
|
|
||||||
gallery.appendChild(a);
|
|
||||||
});
|
|
||||||
const galleryWrapper = document.getElementById("galleryWrapper");
|
|
||||||
galleryWrapper.innerHTML = "";
|
|
||||||
galleryWrapper.appendChild(gallery);
|
|
||||||
|
|
||||||
lightGallery(gallery, {
|
|
||||||
selector: "a",
|
|
||||||
thumbnail: true,
|
|
||||||
zoom: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
// 版本信息
|
|
||||||
const versionBlock = document.getElementById("versionTableWrapper");
|
|
||||||
if (item.changelog?.length > 0) {
|
|
||||||
let table = `<table><tr><th>日期</th><th>版本</th><th>描述</th></tr>`;
|
|
||||||
item.changelog.forEach((row) => {
|
|
||||||
table += `<tr><td>${row.date}</td><td>${row.version}</td><td>${
|
|
||||||
row.description || ""
|
|
||||||
}</td></tr>`;
|
|
||||||
});
|
|
||||||
table += `</table>`;
|
|
||||||
versionBlock.innerHTML = table;
|
|
||||||
} else {
|
|
||||||
versionBlock.innerHTML = "";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function handleSearch() {
|
|
||||||
const keyword = document
|
|
||||||
.getElementById("searchInput")
|
|
||||||
.value.trim()
|
|
||||||
.toLowerCase();
|
|
||||||
const filtered = allData.filter((item) =>
|
|
||||||
item.title.toLowerCase().includes(keyword)
|
|
||||||
);
|
|
||||||
renderList(filtered);
|
|
||||||
}
|
|
||||||
|
|
||||||
fetch("data/kenney_data_local.json")
|
|
||||||
.then((res) => res.json())
|
|
||||||
.then((data) => {
|
|
||||||
allData = data;
|
|
||||||
renderList(data);
|
|
||||||
document
|
|
||||||
.getElementById("searchInput")
|
|
||||||
.addEventListener("input", handleSearch);
|
|
||||||
});
|
|
||||||
@@ -1,115 +0,0 @@
|
|||||||
body {
|
|
||||||
margin: 0;
|
|
||||||
font-family: "Segoe UI", sans-serif;
|
|
||||||
background-color: #f0f2f5;
|
|
||||||
}
|
|
||||||
|
|
||||||
.container {
|
|
||||||
display: flex;
|
|
||||||
height: 100vh;
|
|
||||||
}
|
|
||||||
|
|
||||||
.sidebar {
|
|
||||||
width: 320px;
|
|
||||||
background: #fff;
|
|
||||||
border-right: 1px solid #ddd;
|
|
||||||
padding: 1rem;
|
|
||||||
overflow-y: auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.sidebar input {
|
|
||||||
width: 100%;
|
|
||||||
padding: 0.5rem;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
font-size: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.sidebar ul {
|
|
||||||
list-style: none;
|
|
||||||
padding: 0;
|
|
||||||
margin: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.sidebar li {
|
|
||||||
padding: 0.5rem;
|
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
cursor: pointer;
|
|
||||||
border-radius: 6px;
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 1rem;
|
|
||||||
transition: background 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.sidebar li:hover,
|
|
||||||
.sidebar li.active {
|
|
||||||
background-color: #e6f7ff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.sidebar img.thumb {
|
|
||||||
width: 48px;
|
|
||||||
height: 48px;
|
|
||||||
object-fit: cover;
|
|
||||||
border-radius: 4px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.content {
|
|
||||||
flex: 1;
|
|
||||||
padding: 2rem;
|
|
||||||
overflow-y: auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.placeholder {
|
|
||||||
font-size: 1.2rem;
|
|
||||||
color: #999;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hidden {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
#galleryWrapper {
|
|
||||||
margin-top: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
#galleryWrapper a img {
|
|
||||||
height: 120px;
|
|
||||||
margin: 5px;
|
|
||||||
object-fit: cover;
|
|
||||||
border-radius: 4px;
|
|
||||||
}
|
|
||||||
|
|
||||||
#detailTags {
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
color: #666;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
#downloadBlock {
|
|
||||||
margin: 1rem 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.download-btn {
|
|
||||||
padding: 0.5rem 1rem;
|
|
||||||
background-color: #4caf50;
|
|
||||||
color: white;
|
|
||||||
border-radius: 6px;
|
|
||||||
text-decoration: none;
|
|
||||||
font-size: 0.95rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.download-btn:hover {
|
|
||||||
background-color: #45a049;
|
|
||||||
}
|
|
||||||
|
|
||||||
#versionTableWrapper table {
|
|
||||||
margin-top: 1rem;
|
|
||||||
width: 100%;
|
|
||||||
border-collapse: collapse;
|
|
||||||
}
|
|
||||||
|
|
||||||
#versionTableWrapper th,
|
|
||||||
#versionTableWrapper td {
|
|
||||||
padding: 0.5rem;
|
|
||||||
border: 1px solid #ccc;
|
|
||||||
}
|
|
||||||
364
main.py
Normal file
364
main.py
Normal file
@@ -0,0 +1,364 @@
|
|||||||
|
import os
|
||||||
|
import re
|
||||||
|
import requests
|
||||||
|
import random
|
||||||
|
import time
|
||||||
|
import json
|
||||||
|
from datetime import datetime
|
||||||
|
from mysql_helper import MySQLHelper
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from tqdm import tqdm, trange
|
||||||
|
|
||||||
|
KENNEY_ASSET_URL = "https://www.kenney.nl/assets/"
|
||||||
|
|
||||||
|
def get_headers():
|
||||||
|
"""生成随机 UA 的请求头"""
|
||||||
|
# 一些常见的桌面浏览器 UA 列表（可以自己扩充）
|
||||||
|
USER_AGENTS = [
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||||
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||||
|
"Chrome/126.0.0.0 Safari/537.36",
|
||||||
|
|
||||||
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||||
|
"AppleWebKit/605.1.15 (KHTML, like Gecko) "
|
||||||
|
"Version/17.3 Safari/605.1.15",
|
||||||
|
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
|
||||||
|
"Gecko/20100101 Firefox/128.0"
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"User-Agent": random.choice(USER_AGENTS),
|
||||||
|
"Accept": (
|
||||||
|
"text/html,application/xhtml+xml,application/xml;"
|
||||||
|
"q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"
|
||||||
|
),
|
||||||
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||||
|
"Accept-Encoding": "gzip, deflate, br",
|
||||||
|
"Connection": "keep-alive",
|
||||||
|
"Referer": "https://www.google.com/",
|
||||||
|
"Upgrade-Insecure-Requests": "1"
|
||||||
|
}
|
||||||
|
|
||||||
|
class SmartCrawler:
|
||||||
|
def __init__(self, delay_range=(1, 3)):
|
||||||
|
self.session = requests.Session()
|
||||||
|
self.delay_range = delay_range
|
||||||
|
self.headers = get_headers()
|
||||||
|
|
||||||
|
def get(self, url):
|
||||||
|
time.sleep(random.uniform(*self.delay_range))
|
||||||
|
return self.session.get(url, headers=self.headers)
|
||||||
|
|
||||||
|
def parse_date(date_str):
|
||||||
|
"""将 dd/mm/yyyy 转为 yyyy-mm-dd 格式"""
|
||||||
|
return datetime.strptime(date_str, "%d/%m/%Y").strftime("%Y-%m-%d")
|
||||||
|
|
||||||
|
def get_total_pages() -> int:
|
||||||
|
"""Get total asset pages"""
|
||||||
|
crawler = SmartCrawler()
|
||||||
|
resp = crawler.get(KENNEY_ASSET_URL)
|
||||||
|
soup = BeautifulSoup(resp.text, "lxml")
|
||||||
|
result = 0
|
||||||
|
last_page_button_selector = "#content > section > div > div.row.text-center.margin-top > div > ul > li:last-child > a"
|
||||||
|
last_page_button = soup.select_one(last_page_button_selector)
|
||||||
|
# Should obtain element like this:
|
||||||
|
# <a class="icon" href="https://www.kenney.nl/assets/page:13"> ...
|
||||||
|
# obtain the href attribute, and extract the number after "page:"
|
||||||
|
if last_page_button and "href" in last_page_button.attrs:
|
||||||
|
page_link = last_page_button["href"]
|
||||||
|
splitted_page_link = page_link.split(':')
|
||||||
|
if splitted_page_link[-1].isdigit():
|
||||||
|
result = int(splitted_page_link[-1])
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Expected int in page number, got {splitted_page_link[-1]}")
|
||||||
|
else:
|
||||||
|
raise Exception("Last page button or its href attribute not found")
|
||||||
|
return result
|
||||||
|
|
||||||
|
def get_all_asset_infos(total_pages: int) -> list[dict[str, str]]:
|
||||||
|
result = []
|
||||||
|
for page in trange(1, total_pages + 1, desc="Fetching all assets' page links"):
|
||||||
|
asset_page_url = KENNEY_ASSET_URL + f"page:{page}?search=&sort=release"
|
||||||
|
crawler = SmartCrawler(delay_range=(1, 3))
|
||||||
|
resp = crawler.get(asset_page_url)
|
||||||
|
soup = BeautifulSoup(resp.text, "lxml")
|
||||||
|
contents_selector = "#content > section > div > div:nth-of-type(1)"
|
||||||
|
contents_div = soup.select_one(contents_selector)
|
||||||
|
if contents_div:
|
||||||
|
item_divs = contents_div.find_all("div", recursive=False)
|
||||||
|
for item_div in item_divs:
|
||||||
|
# We may get these info from the grid
|
||||||
|
asset_info = {
|
||||||
|
"name": "",
|
||||||
|
"category": "",
|
||||||
|
"series": "",
|
||||||
|
"page_link": ""
|
||||||
|
}
|
||||||
|
h2_tag = item_div.find("h2")
|
||||||
|
asset_info["name"] = h2_tag.text if h2_tag else None
|
||||||
|
a_tags = item_div.find_all("a")
|
||||||
|
asset_info["category"] = a_tags[2].text if len(a_tags) > 2 else None
|
||||||
|
asset_info["series"] = a_tags[3].text if len(a_tags) > 3 else None
|
||||||
|
asset_info["page_link"] = a_tags[1]["href"] if "href" in a_tags[0].attrs else None
|
||||||
|
result.append(asset_info)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def get_asset_pack_info(asset: dict[str, ]) -> None:
|
||||||
|
crawler = SmartCrawler()
|
||||||
|
resp = crawler.get(asset["page_link"])
|
||||||
|
soup = BeautifulSoup(resp.text, "lxml")
|
||||||
|
|
||||||
|
properties = {}
|
||||||
|
prop_table = soup.select_one("#content > section > div > div > div.col-md-6.text-left > table:nth-of-type(1) > tbody")
|
||||||
|
if prop_table:
|
||||||
|
for row in prop_table.find_all("tr"):
|
||||||
|
cols = row.find_all('td')
|
||||||
|
if len(cols) == 2:
|
||||||
|
key = cols[0].text.strip().rstrip(':')
|
||||||
|
value_links = cols[1].find_all('a')
|
||||||
|
if value_links:
|
||||||
|
value = [a.text.strip() for a in value_links]
|
||||||
|
else:
|
||||||
|
value = cols[1].text.strip()
|
||||||
|
properties[key] = value
|
||||||
|
asset["tags"] = properties.get("Tags", [])
|
||||||
|
|
||||||
|
zip_link = None
|
||||||
|
for a_tag in soup.find_all("a", href=True):
|
||||||
|
href = a_tag["href"]
|
||||||
|
if href.endswith(".zip"):
|
||||||
|
zip_link = "https://www.kenney.nl" + \
|
||||||
|
href if href.startswith("/") else href
|
||||||
|
break
|
||||||
|
|
||||||
|
changelog = []
|
||||||
|
update_table = soup.select_one(
|
||||||
|
'#content > section > div > div > div:nth-of-type(1) > table:nth-of-type(2) > tbody')
|
||||||
|
if update_table:
|
||||||
|
for idx, row in enumerate(update_table.find_all('tr')):
|
||||||
|
cols = row.find_all('td')
|
||||||
|
if len(cols) == 2:
|
||||||
|
date = cols[0].text.strip()
|
||||||
|
spans = cols[1].find_all('span')
|
||||||
|
version = spans[0].text.strip() if len(spans) >= 1 else ''
|
||||||
|
description = spans[1].text.strip() if len(spans) >= 2 else ''
|
||||||
|
version_info = {
|
||||||
|
'date': parse_date(date),
|
||||||
|
'version': version,
|
||||||
|
'description': description,
|
||||||
|
'files': 0,
|
||||||
|
'feat_animation': False,
|
||||||
|
'feat_variation': False,
|
||||||
|
'orig_file_link': None
|
||||||
|
}
|
||||||
|
if idx == 0:
|
||||||
|
# Latest version
|
||||||
|
version_info["files"] = int(''.join(ch for ch in properties["Files"] if ch.isdigit())) if "Files" in properties else 0
|
||||||
|
version_info["feat_animation"] = "Animation" in properties["Features"]
|
||||||
|
version_info["feat_variation"] = "Variation" in properties["Features"]
|
||||||
|
version_info["orig_file_link"] = zip_link if zip_link else None
|
||||||
|
changelog.append(version_info)
|
||||||
|
changelog.reverse()
|
||||||
|
asset["changelog"] = changelog
|
||||||
|
asset["released_at"] = changelog[-1]["date"]
|
||||||
|
asset["updated_at"] = changelog[0]["date"]
|
||||||
|
|
||||||
|
images = []
|
||||||
|
# 封面图（Cover）
|
||||||
|
cover_img = soup.select_one(
|
||||||
|
'#content > section > div > div > div:nth-of-type(2) > a > img')
|
||||||
|
if cover_img and cover_img.get("src"):
|
||||||
|
cover_url = cover_img["src"]
|
||||||
|
if cover_url.startswith("/"):
|
||||||
|
cover_url = "https://www.kenney.nl" + cover_url
|
||||||
|
images.append(cover_url)
|
||||||
|
# 图集中的图像
|
||||||
|
gallery_divs = soup.select(
|
||||||
|
'#content > section > div > div > div:nth-of-type(2) > div > div')
|
||||||
|
for div in gallery_divs:
|
||||||
|
img_tag = div.select_one("a > img")
|
||||||
|
if img_tag and img_tag.get("src"):
|
||||||
|
img_url = img_tag["src"]
|
||||||
|
if img_url.startswith("/"):
|
||||||
|
img_url = "https://www.kenney.nl" + img_url
|
||||||
|
images.append(img_url)
|
||||||
|
asset['images'] = images
|
||||||
|
|
||||||
|
def sync_table(
|
||||||
|
db,
|
||||||
|
table_name: str,
|
||||||
|
column_name: str,
|
||||||
|
items: set[str],
|
||||||
|
) -> dict[str, int]:
|
||||||
|
"""同步唯一字段数据到指定表，并返回 name -> id 的映射"""
|
||||||
|
# 从数据库读取已存在的记录
|
||||||
|
saved_records = db.fetch_all(f"SELECT * FROM {table_name}")
|
||||||
|
saved_names = {r[column_name] for r in saved_records}
|
||||||
|
|
||||||
|
# 找出缺失项
|
||||||
|
missing_items = sorted(items - saved_names)
|
||||||
|
|
||||||
|
# 插入缺失项（假设 id 是自增，不需要手动计算）
|
||||||
|
if missing_items:
|
||||||
|
insert_sql = f"INSERT INTO {table_name} ({column_name}) VALUES (%s)"
|
||||||
|
with db.get_conn() as conn:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.executemany(insert_sql, [(name,) for name in missing_items])
|
||||||
|
conn.commit()
|
||||||
|
cursor.close()
|
||||||
|
|
||||||
|
# 重新获取完整映射（保证 ID 正确）
|
||||||
|
final_records = db.fetch_all(f"SELECT * FROM {table_name}")
|
||||||
|
return {r[column_name]: r["id"] for r in final_records}
|
||||||
|
|
||||||
|
def build_id_map(db, table, name_field, values):
|
||||||
|
return sync_table(db, table, name_field, {v for v in values if v})
|
||||||
|
|
||||||
|
def build_insert_sql(table: str, columns: list[str]):
|
||||||
|
cols_str = ', '.join(columns)
|
||||||
|
placeholders = ', '.join(['%s'] * len(columns))
|
||||||
|
return f"INSERT INTO {table} ({cols_str}) VALUES ({placeholders})"
|
||||||
|
|
||||||
|
allowed_path_pattern = re.compile(r'[^a-zA-Z0-9._-]')
|
||||||
|
def sanitize_path(path: str):
|
||||||
|
if not path:
|
||||||
|
return None
|
||||||
|
return '_'.join([allowed_path_pattern.sub('', word.lower()) for word in path.split()])
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
# total_pages = get_total_pages()
|
||||||
|
# all_asset_infos = get_all_asset_infos(total_pages)
|
||||||
|
|
||||||
|
# # Let the oldest become first in the array to make sure it can be inserted into database first
|
||||||
|
# all_asset_infos.reverse()
|
||||||
|
|
||||||
|
# for asset in tqdm(all_asset_infos, "Fetching asset pack info"):
|
||||||
|
# get_asset_pack_info(asset)
|
||||||
|
|
||||||
|
all_asset_infos: list[dict[str, ]] = json.load(open("all_asset_infos_detailed.json"))
|
||||||
|
# for asset in all_asset_infos:
|
||||||
|
# asset["name"] = None if asset["name"] == "" else asset["name"]
|
||||||
|
# asset["category"] = None if asset["category"] == "" else asset["category"]
|
||||||
|
# asset["series"] = None if asset["series"] == "" else asset["series"]
|
||||||
|
# asset["page_link"] = None if asset["page_link"] == "" else asset["page_link"]
|
||||||
|
# # for log in asset["changelog"]:
|
||||||
|
# # log['files'] = 0 if "files" not in log else log["files"]
|
||||||
|
# # log['feat_animation'] = False if "feat_animation" not in log else log["feat_animation"]
|
||||||
|
# # log['feat_variation'] = False if "feat_variation" not in log else log["feat_variation"]
|
||||||
|
# # log['orig_file_link'] = asset["download"] if log["files"] != 0 else None
|
||||||
|
# # asset.pop('download')
|
||||||
|
# json.dump(all_asset_infos, open("all_asset_infos_detailed_fix.json", "w"))
|
||||||
|
# exit()
|
||||||
|
|
||||||
|
# Download file and save to database
|
||||||
|
output_dir = "media"
|
||||||
|
|
||||||
|
for asset in tqdm(all_asset_infos, "Downloading assets and images"):
|
||||||
|
asset_name = sanitize_path(asset["name"])
|
||||||
|
asset_category = sanitize_path(asset["category"])
|
||||||
|
asset_version = asset["changelog"][-1]["version"]
|
||||||
|
asset_dir_path = os.path.join(output_dir, asset_category)
|
||||||
|
asset_dir_path = os.path.join(asset_dir_path, asset_name)
|
||||||
|
asset["base_asset_path"] = asset_dir_path
|
||||||
|
os.makedirs(asset_dir_path, exist_ok=True)
|
||||||
|
filename = f"{asset_name} V{asset_version}.zip"
|
||||||
|
filepath = os.path.join(asset_dir_path, filename)
|
||||||
|
if os.path.exists(filepath):
|
||||||
|
print(f"✅ {filename} exists, skipping...")
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
download_url = asset["changelog"][-1]["orig_file_link"]
|
||||||
|
with requests.get(download_url, headers=get_headers()) as resp:
|
||||||
|
resp.raise_for_status()
|
||||||
|
with open(filepath, "wb") as f:
|
||||||
|
for chunk in resp.iter_content(chunk_size=8192):
|
||||||
|
if chunk:
|
||||||
|
f.write(chunk)
|
||||||
|
print(f"✅ {filename} download completed.")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"❌ Download failed: {filename} - {e}")
|
||||||
|
time.sleep(random.uniform(1.5, 4.0))
|
||||||
|
# Download asset file
|
||||||
|
exit()
|
||||||
|
|
||||||
|
# Initialize database
|
||||||
|
db = MySQLHelper(
|
||||||
|
"10.147.20.103",
|
||||||
|
"kenney-assets",
|
||||||
|
"9a77caa2a5c705db7e8a93c6a3fbc46a",
|
||||||
|
"kenney_assets"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Build mapping tables
|
||||||
|
category_id_map = build_id_map(db, "category", "name", (a["category"] for a in all_asset_infos))
|
||||||
|
series_id_map = build_id_map(db, "series", "name", (a["series"] for a in all_asset_infos))
|
||||||
|
tags_id_map = build_id_map(db, "tag", "name", (tag for a in all_asset_infos for tag in (a.get("tags") or [])))
|
||||||
|
|
||||||
|
asset_pack_sql = build_insert_sql("asset_pack", ["name", "category_id", "series_id", "released_at", "updated_at", "base_asset_path", "orig_page_link"])
|
||||||
|
asset_pack_values: list[tuple] = []
|
||||||
|
# 给每个 asset 添加 category_id 和 series_id
|
||||||
|
for asset in all_asset_infos:
|
||||||
|
asset["category_id"] = category_id_map.get(asset.get("category"))
|
||||||
|
asset["series_id"] = series_id_map.get(asset.get("series"))
|
||||||
|
asset_pack_values.append((
|
||||||
|
asset["name"],
|
||||||
|
asset["category_id"],
|
||||||
|
asset["series_id"],
|
||||||
|
asset["released_at"],
|
||||||
|
asset["updated_at"],
|
||||||
|
asset["base_asset_path"],
|
||||||
|
asset["page_link"]
|
||||||
|
))
|
||||||
|
db.bulk_insert(asset_pack_sql, asset_pack_values)
|
||||||
|
|
||||||
|
# Get asset_pack id map
|
||||||
|
asset_pack_records = db.fetch_all(f"SELECT id, name FROM asset_pack")
|
||||||
|
asset_pack_id_map = {r["name"]: r["id"] for r in asset_pack_records}
|
||||||
|
|
||||||
|
asset_pack_tag_sql = build_insert_sql("asset_pack_tag", ["asset_pack_id", "tag_id"])
|
||||||
|
asset_pack_tag_values: list[tuple] = []
|
||||||
|
|
||||||
|
update_log_sql = build_insert_sql("update_log", ["asset_pack_id", "released_date", "version", "description", "files_count", "feat_animations", "feat_variations", "orig_download_link"])
|
||||||
|
update_log_values: list[tuple] = []
|
||||||
|
|
||||||
|
asset_pack_image_sql = build_insert_sql("asset_pack_image", ["asset_pack_id", "orig_file_link"])
|
||||||
|
asset_pack_image_values: list[tuple] = []
|
||||||
|
|
||||||
|
for asset in tqdm(all_asset_infos, "Preparing data to database"):
|
||||||
|
asset_pack_id = asset_pack_id_map.get(asset.get("name"))
|
||||||
|
tags = asset.get("tags")
|
||||||
|
for tag in tags:
|
||||||
|
asset_pack_tag_values.append((
|
||||||
|
asset_pack_id,
|
||||||
|
tags_id_map[tag]
|
||||||
|
))
|
||||||
|
changelog = asset.get("changelog")
|
||||||
|
for log in changelog:
|
||||||
|
update_log_values.append((
|
||||||
|
asset_pack_id,
|
||||||
|
log["date"],
|
||||||
|
log["version"],
|
||||||
|
log["description"],
|
||||||
|
log["files"],
|
||||||
|
log["feat_animation"],
|
||||||
|
log["feat_variation"],
|
||||||
|
log["orig_file_link"]
|
||||||
|
))
|
||||||
|
images = asset.get("images")
|
||||||
|
for image in images:
|
||||||
|
asset_pack_image_values.append((
|
||||||
|
asset_pack_id,
|
||||||
|
image
|
||||||
|
))
|
||||||
|
|
||||||
|
with db.get_conn() as conn:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.executemany(asset_pack_tag_sql, asset_pack_tag_values)
|
||||||
|
cursor.executemany(update_log_sql, update_log_values)
|
||||||
|
cursor.executemany(asset_pack_image_sql, asset_pack_image_values)
|
||||||
|
conn.commit()
|
||||||
|
cursor.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
145
mysql_helper.py
Normal file
145
mysql_helper.py
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
from mysql.connector import pooling, Error
|
||||||
|
import logging
|
||||||
|
from contextlib import contextmanager
|
||||||
|
import time
|
||||||
|
|
||||||
|
# ========== 日志配置 ==========
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s [%(levelname)s] %(message)s"
|
||||||
|
)
|
||||||
|
logger = logging.getLogger("MySQLHelper")
|
||||||
|
|
||||||
|
class MySQLHelper:
|
||||||
|
def __init__(self, host, user, password, database, pool_size=5):
|
||||||
|
"""
|
||||||
|
初始化 MySQL 连接池
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
self.pool = pooling.MySQLConnectionPool(
|
||||||
|
pool_name="mypool",
|
||||||
|
pool_size=pool_size,
|
||||||
|
pool_reset_session=True,
|
||||||
|
host=host,
|
||||||
|
user=user,
|
||||||
|
password=password,
|
||||||
|
database=database,
|
||||||
|
charset="utf8mb4"
|
||||||
|
)
|
||||||
|
logger.info("✅ MySQL 连接池已创建，大小=%s", pool_size)
|
||||||
|
except Error as e:
|
||||||
|
logger.error("❌ 创建连接池失败: %s", e)
|
||||||
|
raise
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def get_conn(self):
|
||||||
|
"""
|
||||||
|
获取连接并在使用完毕后释放
|
||||||
|
"""
|
||||||
|
conn = None
|
||||||
|
try:
|
||||||
|
conn = self.pool.get_connection()
|
||||||
|
yield conn
|
||||||
|
except Error as e:
|
||||||
|
logger.error("数据库连接错误: %s", e)
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
if conn:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def execute(self, sql, params=None, commit=False, retry=3):
|
||||||
|
"""
|
||||||
|
执行 INSERT/UPDATE/DELETE 等操作
|
||||||
|
"""
|
||||||
|
for attempt in range(1, retry + 1):
|
||||||
|
try:
|
||||||
|
with self.get_conn() as conn:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute(sql, params or ())
|
||||||
|
if commit:
|
||||||
|
conn.commit()
|
||||||
|
affected = cursor.rowcount
|
||||||
|
cursor.close()
|
||||||
|
return affected
|
||||||
|
except Error as e:
|
||||||
|
logger.warning("执行 SQL 失败 (尝试 %s/%s): %s", attempt, retry, e)
|
||||||
|
time.sleep(1)
|
||||||
|
if attempt == retry:
|
||||||
|
raise
|
||||||
|
|
||||||
|
def fetch_all(self, sql, params=None, retry=3):
|
||||||
|
"""
|
||||||
|
查询多条记录
|
||||||
|
"""
|
||||||
|
for attempt in range(1, retry + 1):
|
||||||
|
try:
|
||||||
|
with self.get_conn() as conn:
|
||||||
|
cursor = conn.cursor(dictionary=True)
|
||||||
|
cursor.execute(sql, params or ())
|
||||||
|
result = cursor.fetchall()
|
||||||
|
cursor.close()
|
||||||
|
return result
|
||||||
|
except Error as e:
|
||||||
|
logger.warning("查询失败 (尝试 %s/%s): %s", attempt, retry, e)
|
||||||
|
time.sleep(1)
|
||||||
|
if attempt == retry:
|
||||||
|
raise
|
||||||
|
|
||||||
|
def fetch_one(self, sql, params=None, retry=3):
|
||||||
|
"""
|
||||||
|
查询单条记录
|
||||||
|
"""
|
||||||
|
result = self.fetch_all(sql, params, retry)
|
||||||
|
return result[0] if result else None
|
||||||
|
|
||||||
|
def bulk_insert(self, sql: str, rows: list[tuple]):
|
||||||
|
with self.get_conn() as conn:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.executemany(sql, rows)
|
||||||
|
conn.commit()
|
||||||
|
cursor.close()
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def transaction(self):
|
||||||
|
"""
|
||||||
|
事务上下文管理器
|
||||||
|
用法:
|
||||||
|
with db.transaction() as cursor:
|
||||||
|
cursor.execute(...)
|
||||||
|
cursor.execute(...)
|
||||||
|
"""
|
||||||
|
with self.get_conn() as conn:
|
||||||
|
try:
|
||||||
|
cursor = conn.cursor()
|
||||||
|
yield cursor
|
||||||
|
conn.commit()
|
||||||
|
except:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
cursor.close()
|
||||||
|
|
||||||
|
# ================= 使用示例 =================
|
||||||
|
if __name__ == "__main__":
|
||||||
|
db = MySQLHelper(
|
||||||
|
host="localhost",
|
||||||
|
user="root",
|
||||||
|
password="123456",
|
||||||
|
database="test_db",
|
||||||
|
pool_size=5
|
||||||
|
)
|
||||||
|
|
||||||
|
# 插入数据
|
||||||
|
db.execute("INSERT INTO users(name, age) VALUES (%s, %s)", ("Alice", 25), commit=True)
|
||||||
|
|
||||||
|
# 查询数据
|
||||||
|
users = db.fetch_all("SELECT * FROM users WHERE age > %s", (18,))
|
||||||
|
logger.info("查询结果: %s", users)
|
||||||
|
|
||||||
|
# 事务示例
|
||||||
|
try:
|
||||||
|
with db.transaction() as cur:
|
||||||
|
cur.execute("UPDATE users SET age = age + 1 WHERE name = %s", ("Alice",))
|
||||||
|
cur.execute("INSERT INTO logs(message) VALUES (%s)", ("Alice age updated",))
|
||||||
|
except Error as e:
|
||||||
|
logger.error("事务失败: %s", e)
|
||||||
73
queries.sql
Normal file
73
queries.sql
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
SET FOREIGN_KEY_CHECKS = 0;
|
||||||
|
DROP TABLE IF EXISTS category;
|
||||||
|
DROP TABLE IF EXISTS series;
|
||||||
|
DROP TABLE IF EXISTS tag;
|
||||||
|
DROP TABLE IF EXISTS asset_pack;
|
||||||
|
DROP TABLE IF EXISTS asset_pack_image;
|
||||||
|
DROP TABLE IF EXISTS asset_pack_tag;
|
||||||
|
DROP TABLE IF EXISTS update_log;
|
||||||
|
SET FOREIGN_KEY_CHECKS = 1;
|
||||||
|
|
||||||
|
CREATE TABLE category (
|
||||||
|
id INT PRIMARY KEY AUTO_INCREMENT,
|
||||||
|
name VARCHAR(100) NOT NULL UNIQUE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE series (
|
||||||
|
id INT PRIMARY KEY AUTO_INCREMENT,
|
||||||
|
name VARCHAR(100) NOT NULL UNIQUE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE tag (
|
||||||
|
id INT PRIMARY KEY AUTO_INCREMENT,
|
||||||
|
name VARCHAR(100) NOT NULL UNIQUE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE asset_pack (
|
||||||
|
id INT PRIMARY KEY AUTO_INCREMENT,
|
||||||
|
name VARCHAR(255) NOT NULL UNIQUE,
|
||||||
|
category_id INT NOT NULL,
|
||||||
|
series_id INT,
|
||||||
|
discovered_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
released_at DATE,
|
||||||
|
updated_at DATE,
|
||||||
|
base_asset_path VARCHAR(255),
|
||||||
|
orig_page_link VARCHAR(255),
|
||||||
|
FOREIGN KEY (category_id) REFERENCES category(id) ON DELETE RESTRICT ON UPDATE CASCADE,
|
||||||
|
FOREIGN KEY (series_id) REFERENCES series(id) ON DELETE SET NULL ON UPDATE CASCADE,
|
||||||
|
INDEX idx_name (name),
|
||||||
|
INDEX idx_category (category_id),
|
||||||
|
INDEX idx_series (series_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE asset_pack_image (
|
||||||
|
id INT PRIMARY KEY AUTO_INCREMENT,
|
||||||
|
asset_pack_id INT NOT NULL,
|
||||||
|
image_file_name VARCHAR(255),
|
||||||
|
orig_file_link VARCHAR(255),
|
||||||
|
FOREIGN KEY (asset_pack_id) REFERENCES asset_pack(id) ON DELETE CASCADE,
|
||||||
|
INDEX idx_asset_pack_id (asset_pack_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE asset_pack_tag (
|
||||||
|
asset_pack_id INT NOT NULL,
|
||||||
|
tag_id INT NOT NULL,
|
||||||
|
PRIMARY KEY (asset_pack_id, tag_id),
|
||||||
|
FOREIGN KEY (asset_pack_id) REFERENCES asset_pack(id) ON DELETE CASCADE,
|
||||||
|
FOREIGN KEY (tag_id) REFERENCES tag(id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE update_log (
|
||||||
|
id INT PRIMARY KEY AUTO_INCREMENT,
|
||||||
|
asset_pack_id INT NOT NULL,
|
||||||
|
released_date DATE NOT NULL,
|
||||||
|
version VARCHAR(20),
|
||||||
|
description VARCHAR(500),
|
||||||
|
files_count INT UNSIGNED DEFAULT 0,
|
||||||
|
feat_animations TINYINT(1) DEFAULT 0,
|
||||||
|
feat_variations TINYINT(1) DEFAULT 0,
|
||||||
|
zip_file_name VARCHAR(255),
|
||||||
|
orig_download_link VARCHAR(255),
|
||||||
|
FOREIGN KEY (asset_pack_id) REFERENCES asset_pack(id) ON DELETE CASCADE,
|
||||||
|
INDEX idx_released_date (released_date)
|
||||||
|
);
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
beautifulsoup4==4.13.4
|
|
||||||
certifi==2025.1.31
|
|
||||||
charset-normalizer==3.4.1
|
|
||||||
colorama==0.4.6
|
|
||||||
idna==3.10
|
|
||||||
lxml==5.3.2
|
|
||||||
requests==2.32.3
|
|
||||||
soupsieve==2.6
|
|
||||||
tqdm==4.67.1
|
|
||||||
typing_extensions==4.13.2
|
|
||||||
urllib3==2.4.0
|
|
||||||
@@ -1,136 +0,0 @@
|
|||||||
import requests
|
|
||||||
import json
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
from tqdm import tqdm, trange
|
|
||||||
|
|
||||||
base_url = "https://www.kenney.nl/assets/page:"
|
|
||||||
total_pages = 13
|
|
||||||
all_links = []
|
|
||||||
|
|
||||||
headers = {
|
|
||||||
"User-Agent": "Mozilla/5.0"
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def parse_resource_page(url):
|
|
||||||
response = requests.get(url, headers=headers)
|
|
||||||
soup = BeautifulSoup(response.text, "lxml")
|
|
||||||
result = {}
|
|
||||||
|
|
||||||
# 资源名称
|
|
||||||
title_tag = soup.select_one(
|
|
||||||
'#content > section > div > div > div:nth-of-type(1) > h1')
|
|
||||||
result['title'] = title_tag.text.strip() if title_tag else 'N/A'
|
|
||||||
|
|
||||||
# 属性表
|
|
||||||
properties = {}
|
|
||||||
prop_table = soup.select_one(
|
|
||||||
'#content > section > div > div > div:nth-of-type(1) > table:nth-of-type(1) > tbody')
|
|
||||||
if prop_table:
|
|
||||||
for row in prop_table.find_all('tr'):
|
|
||||||
cols = row.find_all('td')
|
|
||||||
if len(cols) == 2:
|
|
||||||
key = cols[0].text.strip().rstrip(':')
|
|
||||||
value_links = cols[1].find_all('a')
|
|
||||||
if value_links:
|
|
||||||
value = [a.text.strip() for a in value_links]
|
|
||||||
else:
|
|
||||||
value = cols[1].text.strip()
|
|
||||||
properties[key] = value
|
|
||||||
result['properties'] = properties
|
|
||||||
|
|
||||||
# 更新记录
|
|
||||||
changelog = []
|
|
||||||
update_table = soup.select_one(
|
|
||||||
'#content > section > div > div > div:nth-of-type(1) > table:nth-of-type(2) > tbody')
|
|
||||||
if update_table:
|
|
||||||
for row in update_table.find_all('tr'):
|
|
||||||
cols = row.find_all('td')
|
|
||||||
if len(cols) == 2:
|
|
||||||
date = cols[0].text.strip()
|
|
||||||
spans = cols[1].find_all('span')
|
|
||||||
version = spans[0].text.strip() if len(spans) >= 1 else ''
|
|
||||||
description = spans[1].text.strip() if len(spans) >= 2 else ''
|
|
||||||
changelog.append({
|
|
||||||
'date': date,
|
|
||||||
'version': version,
|
|
||||||
'description': description
|
|
||||||
})
|
|
||||||
result['changelog'] = changelog
|
|
||||||
|
|
||||||
# 下载链接
|
|
||||||
zip_link = None
|
|
||||||
for a_tag in soup.find_all("a", href=True):
|
|
||||||
href = a_tag["href"]
|
|
||||||
if href.endswith(".zip"):
|
|
||||||
zip_link = "https://www.kenney.nl" + \
|
|
||||||
href if href.startswith("/") else href
|
|
||||||
break
|
|
||||||
result['download'] = zip_link if zip_link else "N/A"
|
|
||||||
|
|
||||||
# ✅ 图片提取
|
|
||||||
images = []
|
|
||||||
|
|
||||||
# 封面图（Cover）
|
|
||||||
cover_img = soup.select_one(
|
|
||||||
'#content > section > div > div > div:nth-of-type(2) > a > img')
|
|
||||||
if cover_img and cover_img.get("src"):
|
|
||||||
cover_url = cover_img["src"]
|
|
||||||
if cover_url.startswith("/"):
|
|
||||||
cover_url = "https://www.kenney.nl" + cover_url
|
|
||||||
images.append(cover_url)
|
|
||||||
|
|
||||||
# 图集中的图像
|
|
||||||
gallery_divs = soup.select(
|
|
||||||
'#content > section > div > div > div:nth-of-type(2) > div > div')
|
|
||||||
for div in gallery_divs:
|
|
||||||
img_tag = div.select_one("a > img")
|
|
||||||
if img_tag and img_tag.get("src"):
|
|
||||||
img_url = img_tag["src"]
|
|
||||||
if img_url.startswith("/"):
|
|
||||||
img_url = "https://www.kenney.nl" + img_url
|
|
||||||
images.append(img_url)
|
|
||||||
|
|
||||||
result['images'] = images
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# for page in range(1, total_pages + 1):
|
|
||||||
for page in trange(1, total_pages + 1, desc="Fetching all assets' page links"):
|
|
||||||
url = base_url + str(page)
|
|
||||||
response = requests.get(url, headers=headers)
|
|
||||||
soup = BeautifulSoup(response.text, "lxml")
|
|
||||||
|
|
||||||
# 定位到//*[@id="content"]/section/div/div[1]
|
|
||||||
content_div = soup.select_one(
|
|
||||||
"#content > section > div > div:nth-of-type(1)")
|
|
||||||
|
|
||||||
if content_div:
|
|
||||||
item_divs = content_div.find_all("div", recursive=False)
|
|
||||||
|
|
||||||
for item_div in item_divs:
|
|
||||||
a_tag = item_div.find("a")
|
|
||||||
if a_tag and "href" in a_tag.attrs:
|
|
||||||
link = a_tag["href"]
|
|
||||||
full_link = link
|
|
||||||
all_links.append(full_link)
|
|
||||||
|
|
||||||
print(f"总共提取到 {len(all_links)} 个链接 ✅")
|
|
||||||
|
|
||||||
with open("kenney_links.txt", "w", encoding="utf-8") as f:
|
|
||||||
for link in all_links:
|
|
||||||
f.write(link + "\n")
|
|
||||||
|
|
||||||
# 爬取页面内的信息
|
|
||||||
all_resource_data = []
|
|
||||||
|
|
||||||
# for link in all_links:
|
|
||||||
for link in tqdm(all_links, desc="Fetching all assets' data"):
|
|
||||||
resource_data = parse_resource_page(link)
|
|
||||||
all_resource_data.append(resource_data)
|
|
||||||
|
|
||||||
with open("kenney_data.json", "w", encoding="utf-8") as f:
|
|
||||||
json.dump(all_resource_data, f, ensure_ascii=False, indent=4)
|
|
||||||
|
|
||||||
print("数据爬取完成 ✅")
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
import os
|
|
||||||
import json
|
|
||||||
import requests
|
|
||||||
import time
|
|
||||||
import random
|
|
||||||
from tqdm import tqdm
|
|
||||||
|
|
||||||
# === 配置路径 ===
|
|
||||||
json_path = "kenney_data.json" # JSON 数据路径
|
|
||||||
output_dir = "kenney_assets" # 下载根目录
|
|
||||||
|
|
||||||
# === 加载 JSON 数据 ===
|
|
||||||
with open(json_path, "r", encoding="utf-8") as f:
|
|
||||||
resources = json.load(f)
|
|
||||||
|
|
||||||
# === 工具函数 ===
|
|
||||||
|
|
||||||
|
|
||||||
def sanitize_filename(name):
|
|
||||||
return "".join(c for c in name if c.isalnum() or c in "._- ()").strip()
|
|
||||||
|
|
||||||
|
|
||||||
def download_zip(entry):
|
|
||||||
title = entry["title"]
|
|
||||||
version = entry["changelog"][0]["version"] if entry["changelog"] else "1.0"
|
|
||||||
download_url = entry.get("download")
|
|
||||||
|
|
||||||
# 提取分类、系列
|
|
||||||
category = entry["properties"].get("Category", ["Uncategorized"])[0]
|
|
||||||
series = entry["properties"].get("Series", [None])[0]
|
|
||||||
|
|
||||||
# 构建目录结构
|
|
||||||
folder_path = os.path.join(output_dir, sanitize_filename(category))
|
|
||||||
if series:
|
|
||||||
folder_path = os.path.join(folder_path, sanitize_filename(series))
|
|
||||||
os.makedirs(folder_path, exist_ok=True)
|
|
||||||
|
|
||||||
# 构建文件路径
|
|
||||||
filename = f"{sanitize_filename(title)} V{version}.zip"
|
|
||||||
filepath = os.path.join(folder_path, filename)
|
|
||||||
|
|
||||||
if os.path.exists(filepath):
|
|
||||||
print(f"✅ 已存在，跳过: {filename}")
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
|
||||||
print(f"⬇️ 开始下载: {filename}")
|
|
||||||
with requests.get(download_url, stream=True, timeout=60) as r:
|
|
||||||
r.raise_for_status()
|
|
||||||
with open(filepath, "wb") as f:
|
|
||||||
for chunk in r.iter_content(chunk_size=8192):
|
|
||||||
if chunk:
|
|
||||||
f.write(chunk)
|
|
||||||
print(f"✅ 下载完成: {filename}")
|
|
||||||
except Exception as e:
|
|
||||||
print(f"❌ 下载失败: {filename} - {e}")
|
|
||||||
|
|
||||||
# 模拟人类行为：随机等待
|
|
||||||
time.sleep(random.uniform(1.5, 4.0))
|
|
||||||
|
|
||||||
|
|
||||||
# === 启动批量下载 ===
|
|
||||||
idx = 0
|
|
||||||
for resource in tqdm(resources, desc="处理资源"):
|
|
||||||
if idx < 156:
|
|
||||||
idx += 1
|
|
||||||
continue
|
|
||||||
if "download" in resource and resource["download"].endswith(".zip"):
|
|
||||||
download_zip(resource)
|
|
||||||
idx += 1
|
|
||||||
|
|
||||||
print("\n✅ 所有资源处理完成")
|
|
||||||
@@ -1,61 +0,0 @@
|
|||||||
import os
|
|
||||||
import json
|
|
||||||
import requests
|
|
||||||
import time
|
|
||||||
import random
|
|
||||||
from urllib.parse import urlparse, unquote
|
|
||||||
from tqdm import tqdm
|
|
||||||
|
|
||||||
# ========== 配置 ==========
|
|
||||||
json_path = "kenney_data.json" # JSON 数据路径
|
|
||||||
output_root = "kenney_assets_images" # 存储根目录
|
|
||||||
headers = {"User-Agent": "Mozilla/5.0"}
|
|
||||||
|
|
||||||
# ========== 工具函数 ==========
|
|
||||||
|
|
||||||
|
|
||||||
def sanitize_filename(name):
|
|
||||||
return "".join(c for c in name if c.isalnum() or c in "._- ()").strip()
|
|
||||||
|
|
||||||
|
|
||||||
def download_image(url, save_path):
|
|
||||||
if os.path.exists(save_path):
|
|
||||||
print(f"✅ 已存在，跳过: {save_path}")
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
response = requests.get(url, stream=True, timeout=30)
|
|
||||||
with open(save_path, "wb") as f:
|
|
||||||
for chunk in response.iter_content(chunk_size=8192):
|
|
||||||
f.write(chunk)
|
|
||||||
print(f"✅ 下载完成: {save_path}")
|
|
||||||
except Exception as e:
|
|
||||||
print(f"❌ 下载失败: {url} - {e}")
|
|
||||||
time.sleep(random.uniform(1.5, 4.0)) # 模拟人类访问
|
|
||||||
|
|
||||||
|
|
||||||
# ========== 加载 JSON ==========
|
|
||||||
with open(json_path, "r", encoding="utf-8") as f:
|
|
||||||
resources = json.load(f)
|
|
||||||
|
|
||||||
# ========== 批量处理 ==========
|
|
||||||
for entry in tqdm(resources, desc="处理资源"):
|
|
||||||
title = entry["title"]
|
|
||||||
category = entry["properties"].get("Category", ["Uncategorized"])[0]
|
|
||||||
series = entry["properties"].get("Series", [None])[0]
|
|
||||||
images = entry.get("images", [])
|
|
||||||
|
|
||||||
# 构建路径：Category/Series/Title/
|
|
||||||
path = os.path.join(output_root, sanitize_filename(category))
|
|
||||||
if series:
|
|
||||||
path = os.path.join(path, sanitize_filename(series))
|
|
||||||
path = os.path.join(path, sanitize_filename(title))
|
|
||||||
os.makedirs(path, exist_ok=True)
|
|
||||||
|
|
||||||
for img_url in images:
|
|
||||||
parsed_url = urlparse(img_url)
|
|
||||||
img_name = os.path.basename(parsed_url.path)
|
|
||||||
img_name = unquote(img_name) # 处理 URL 编码，如 %20 => 空格
|
|
||||||
img_path = os.path.join(path, img_name)
|
|
||||||
download_image(img_url, img_path)
|
|
||||||
|
|
||||||
print("\n🎉 所有图片处理完成！")
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
import os
|
|
||||||
import json
|
|
||||||
from urllib.parse import urlparse, unquote
|
|
||||||
|
|
||||||
# === 配置路径 ===
|
|
||||||
input_json = "kenney_data.json"
|
|
||||||
output_json = "kenney_data_local.json"
|
|
||||||
|
|
||||||
zip_root = "kenney_assets"
|
|
||||||
img_root = "kenney_assets_images"
|
|
||||||
|
|
||||||
# === 工具函数 ===
|
|
||||||
|
|
||||||
|
|
||||||
def sanitize_filename(name):
|
|
||||||
return "".join(c for c in name if c.isalnum() or c in "._- ()").strip()
|
|
||||||
|
|
||||||
|
|
||||||
def build_zip_path(entry):
|
|
||||||
title = entry["title"]
|
|
||||||
version = entry["changelog"][0]["version"] if entry["changelog"] else "1.0"
|
|
||||||
category = entry["properties"].get("Category", ["Uncategorized"])[0]
|
|
||||||
series = entry["properties"].get("Series", [None])[0]
|
|
||||||
|
|
||||||
folder = os.path.join(zip_root, sanitize_filename(category))
|
|
||||||
if series:
|
|
||||||
folder = os.path.join(folder, sanitize_filename(series))
|
|
||||||
filename = f"{sanitize_filename(title)} V{version}.zip"
|
|
||||||
return os.path.join(folder, filename)
|
|
||||||
|
|
||||||
|
|
||||||
def build_image_paths(entry):
|
|
||||||
title = entry["title"]
|
|
||||||
category = entry["properties"].get("Category", ["Uncategorized"])[0]
|
|
||||||
series = entry["properties"].get("Series", [None])[0]
|
|
||||||
images = entry.get("images", [])
|
|
||||||
|
|
||||||
folder = os.path.join(img_root, sanitize_filename(category))
|
|
||||||
if series:
|
|
||||||
folder = os.path.join(folder, sanitize_filename(series))
|
|
||||||
folder = os.path.join(folder, sanitize_filename(title))
|
|
||||||
|
|
||||||
local_paths = []
|
|
||||||
for img_url in images:
|
|
||||||
parsed = urlparse(img_url)
|
|
||||||
filename = unquote(os.path.basename(parsed.path))
|
|
||||||
local_paths.append(os.path.join(folder, filename))
|
|
||||||
|
|
||||||
return local_paths
|
|
||||||
|
|
||||||
|
|
||||||
# === 主处理 ===
|
|
||||||
with open(input_json, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
for entry in data:
|
|
||||||
if "download" in entry and entry["download"].endswith(".zip"):
|
|
||||||
zip_path = build_zip_path(entry)
|
|
||||||
if os.path.exists(zip_path):
|
|
||||||
entry["download"] = zip_path # 替换为本地路径
|
|
||||||
if "images" in entry and isinstance(entry["images"], list):
|
|
||||||
entry["images"] = build_image_paths(entry)
|
|
||||||
|
|
||||||
# === 保存修改后的 JSON ===
|
|
||||||
with open(output_json, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
||||||
|
|
||||||
print("✅ 已更新 JSON：本地路径写入完毕！")
|
|
||||||
33
思路.txt
33
思路.txt
@@ -1,33 +0,0 @@
|
|||||||
我想要爬取这个网页 https://www.kenney.nl/assets/page:1 下（共有 13 页）
|
|
||||||
//*[@id="content"]/section/div/div[1] 这个元素（里边是 n 个 div 元素）
|
|
||||||
我需要获取每个 Div 元素的
|
|
||||||
//*[@id="content"]/section/div/div[1]/div[1]/div/a 这个 a 元素
|
|
||||||
将它的链接保存成一个列表
|
|
||||||
|
|
||||||
然后依次访问这个列表中的所有页面，
|
|
||||||
|
|
||||||
CSV 格式如下：
|
|
||||||
资源名称, 分类, 系列, 资源数, 标签（有多个）,资源数量
|
|
||||||
|
|
||||||
访问这个界面，然后我需要获取它的所有信息：
|
|
||||||
资源名称：//*[@id="content"]/section/div/div/div[1]/h1
|
|
||||||
|
|
||||||
各种属性：//*[@id="content"]/section/div/div/div[1]/table[1]/tbody
|
|
||||||
上面这个表，他是个两列 N 行的表。
|
|
||||||
表的左侧可以看作是 key: Category, Series, Assets, Variation(s), Tags, License 等……
|
|
||||||
表的右侧是 Value，但它可能会有各种组成方式，下边是常见的：
|
|
||||||
- 普通的 td > a
|
|
||||||
- 只有 td
|
|
||||||
- td 之后有多个 a（如 Tags)
|
|
||||||
|
|
||||||
更新记录：//*[@id="content"]/section/div/div/div[1]/table[2]/tbody
|
|
||||||
他也是个两行 N 列的表，
|
|
||||||
左侧是更新日期 dd/mm/yyyy
|
|
||||||
右侧是 td > 第一个 span 是版本，第二个 span 是更新内容，有时候没有第二个 span
|
|
||||||
|
|
||||||
继续增加 parse_resource_page(url) 函数，
|
|
||||||
我需要获取它的封面图：//*[@id="content"]/section/div/div/div[2]/a/img
|
|
||||||
以及其他图片（如有）
|
|
||||||
//*[@id="content"]/section/div/div/div[2]/div 这里子元素可能有 n 个 div，我需要拿到它的 img
|
|
||||||
//*[@id="content"]/section/div/div/div[2]/div/div[1]/a/img
|
|
||||||
封面图存了之后，也得加在图集的数组里。
|
|
||||||
Reference in New Issue
Block a user