Cao Note

外观

爬虫与 Puppeteer ​

HTTP 请求工具 ​

bash
npm install axios request cheerio
npm install axios request cheerio

request ​

js
const request = require('request')

request('http://example.com', (err, response, body) => {
  // body 为页面内容
  // response 为响应头信息
})
const request = require('request')

request('http://example.com', (err, response, body) => {
  // body 为页面内容
  // response 为响应头信息
})

axios ​

js
const axios = require('axios')

// 下载图片
const ws = fs.createWriteStream('image.jpg')
axios.get('https://example.com/image.jpg', { responseType: 'stream' })
  .then((res) => {
    res.data.pipe(ws)
    res.data.on('close', () => ws.close())
  })
const axios = require('axios')

// 下载图片
const ws = fs.createWriteStream('image.jpg')
axios.get('https://example.com/image.jpg', { responseType: 'stream' })
  .then((res) => {
    res.data.pipe(ws)
    res.data.on('close', () => ws.close())
  })

延迟函数（防拉黑） ​

js
function wait(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms))
}
function wait(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms))
}

cheerio（服务端 jQuery） ​

js
const cheerio = require('cheerio')
const request = require('request')

request('https://example.com', (err, response, body) => {
  const $ = cheerio.load(body)

  $('div.content a').each((index, element) => {
    console.log({
      name: $(element).attr('alt'),
      url: $(element).attr('href')
    })
  })
})
const cheerio = require('cheerio')
const request = require('request')

request('https://example.com', (err, response, body) => {
  const $ = cheerio.load(body)

  $('div.content a').each((index, element) => {
    console.log({
      name: $(element).attr('alt'),
      url: $(element).attr('href')
    })
  })
})

Puppeteer ​

Puppeteer 是一个无头浏览器工具，用于自动化控制 Chrome。

安装 ​

bash
npm install puppeteer-core
npm install puppeteer-core

启动浏览器 ​

js
const puppeteer = require('puppeteer')

puppeteer.launch({
  headless: false,           // true 为无头模式
  defaultViewport: { width: 1400, height: 800 },
  slowMo: 250,               // 减慢操作速度，方便调试
  executablePath: 'C:/Program Files/Google/Chrome/Application/chrome.exe'
}).then(async (browser) => {
  const page = await browser.newPage()
  await page.goto('https://example.com')
})
const puppeteer = require('puppeteer')

puppeteer.launch({
  headless: false,           // true 为无头模式
  defaultViewport: { width: 1400, height: 800 },
  slowMo: 250,               // 减慢操作速度，方便调试
  executablePath: 'C:/Program Files/Google/Chrome/Application/chrome.exe'
}).then(async (browser) => {
  const page = await browser.newPage()
  await page.goto('https://example.com')
})

获取页面内容 ​

js
// page.$$eval 获取多个元素内容（代码在浏览器中执行）
const links = await page.$$eval('#menu li a', (elements) => {
  return elements.map((el) => ({
    href: el.getAttribute('href'),
    text: el.innerText
  }))
})

// page.$eval 获取单个元素
const title = await page.$eval('h1', (el) => el.innerText)

// page.$ 返回 ElementHandle（用于点击操作）
const btn = await page.$('.submit-btn')
await btn.click()
// page.$$eval 获取多个元素内容（代码在浏览器中执行）
const links = await page.$$eval('#menu li a', (elements) => {
  return elements.map((el) => ({
    href: el.getAttribute('href'),
    text: el.innerText
  }))
})

// page.$eval 获取单个元素
const title = await page.$eval('h1', (el) => el.innerText)

// page.$ 返回 ElementHandle（用于点击操作）
const btn = await page.$('.submit-btn')
await btn.click()

模拟用户输入 ​

js
// 方式一：focus + keyboard
const input = await page.$('.search-input')
await input.focus()
await page.keyboard.type('搜索内容')

// 方式二：page.type
await page.type('#kw', 'puppeteer')
await Promise.all([
  page.waitForNavigation(),
  page.click('#search-btn')
])
// 方式一：focus + keyboard
const input = await page.$('.search-input')
await input.focus()
await page.keyboard.type('搜索内容')

// 方式二：page.type
await page.type('#kw', 'puppeteer')
await Promise.all([
  page.waitForNavigation(),
  page.click('#search-btn')
])

拦截网络请求 ​

js
await page.setRequestInterception(true)

page.on('request', (interceptedRequest) => {
  if (interceptedRequest.url().endsWith('.png')) {
    interceptedRequest.abort()  // 中断图片请求
  } else {
    interceptedRequest.continue()
  }
})
await page.setRequestInterception(true)

page.on('request', (interceptedRequest) => {
  if (interceptedRequest.url().endsWith('.png')) {
    interceptedRequest.abort()  // 中断图片请求
  } else {
    interceptedRequest.continue()
  }
})

页面内执行代码 ​

js
// page.evaluate — 返回字符串
const result = await page.evaluate(() => document.title)

// page.evaluateHandle — 返回 JSHandle
const handle = await page.evaluateHandle(() => document.body)
const html = await handle.jsonValue()
await handle.dispose()
// page.evaluate — 返回字符串
const result = await page.evaluate(() => document.title)

// page.evaluateHandle — 返回 JSHandle
const handle = await page.evaluateHandle(() => document.body)
const html = await handle.jsonValue()
await handle.dispose()

常用 API ​

  • page.close() — 关闭当前页面
  • browser.close() — 关闭浏览器
  • page.waitForSelector(selector) — 等待元素出现
  • page.mouse.click(x, y) — 鼠标点击
  • page.on('requestfinished', callback) — 监听请求完成

Copyright © 2024-present xiaocao