【教程】Puppeteer自动化抓取实战：零代码基础也能写的网页数据采集脚本 - 闲社

npm install puppeteer

复制代码

npm config set puppeteer_download_host=https://npmmirror.com/mirrors
npm install puppeteer

复制代码

scraper.js

复制代码

const puppeteer = require('puppeteer');
(async () => {
// 1. 启动浏览器（headless: true 表示无界面，后台跑）
const browser = await puppeteer.launch({ headless: true });
const page = await browser.newPage();
// 2. 设置 User-Agent，模拟真实浏览器
await page.setUserAgent(
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36'
);
// 3. 设置视口，避免被识别为无头浏览器
await page.setViewport({ width: 1920, height: 1080 });
// 4. 打开目标页面
await page.goto('https://news.example.com/list', {
waitUntil: 'networkidle2', // 等网络空闲再开始
timeout: 30000
});
// 5. 提取数据（用 page.evaluate 在页面上下文中执行 JS）
const articles = await page.evaluate(() => {
const items = document.querySelectorAll('.news-item');
return Array.from(items).map(item => ({
title: item.querySelector('h2')?.innerText?.trim(),
link: item.querySelector('a')?.href,
summary: item.querySelector('.summary')?.innerText?.trim(),
date: item.querySelector('.date')?.innerText?.trim()
}));
});
console.log('采集到', articles.length, '条数据');
console.log(JSON.stringify(articles, null, 2));
// 6. 关闭浏览器
await browser.close();
})();

复制代码

node scraper.js

复制代码

const puppeteer = require('puppeteer');
async function scrapeWithRetry(url, maxRetries = 3) {
for (let attempt = 1; attempt {
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] });
});
await page.setUserAgent(
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36'
);
await page.setViewport({ width: 1920, height: 1080 });
// 设置更真实的请求头
await page.setExtraHTTPHeaders({
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
});
await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
// 等 2-5 秒随机延迟，模拟人类阅读
await page.waitForTimeout(2000 + Math.random() * 3000);
const data = await page.evaluate(() => {
const items = document.querySelectorAll('.news-item');
return Array.from(items).map(item => ({
title: item.querySelector('h2')?.innerText?.trim(),
link: item.querySelector('a')?.href,
summary: item.querySelector('.summary')?.innerText?.trim()
})).filter(i => i.title);
});
await browser.close();
return data;
} catch (err) {
console.log(`第 ${attempt} 次尝试失败: ${err.message}`);
await browser.close();
if (attempt === maxRetries) throw err;
await new Promise(r => setTimeout(r, 5000 * attempt)); // 递增延迟
}
}
}
(async () => {
const result = await scrapeWithRetry('https://news.example.com/list');
console.log('最终采集结果:', JSON.stringify(result, null, 2));
})();

复制代码

const fs = require('fs');
// 在 scrapeWithRetry 返回后添加：
fs.writeFileSync('data.json', JSON.stringify(result, null, 2));
console.log('已保存到 data.json');

复制代码

npm install better-sqlite3

复制代码

const Database = require('better-sqlite3');
const db = new Database('articles.db');
db.exec(`
CREATE TABLE IF NOT EXISTS articles (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT,
link TEXT,
summary TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
`);
const insert = db.prepare('INSERT INTO articles (title, link, summary) VALUES (?, ?, ?)');
result.forEach(item => insert.run(item.title, item.link, item.summary));
console.log('已存入数据库');

复制代码