npm install apify --save
復(fù)制代碼

const Apify = require('apify');

Apify.main(async () => {
    const requestQueue = await Apify.openRequestQueue();
    await requestQueue.addRequest({ url: 'https://www.iana.org/' });
    const pseudoUrls = [new Apify.PseudoUrl('https://www.iana.org/[.*]')];

    const crawler = new Apify.PuppeteerCrawler({
        requestQueue,
        handlePageFunction: async ({ request, page }) => {
            const title = await page.title();
            console.log(`Title of ${request.url}: ${title}`);
            await Apify.utils.enqueueLinks({
                page,
                selector: 'a',
                pseudoUrls,
                requestQueue,
            });
        },
        maxRequestsPerCrawl: 100,
        maxConcurrency: 10,
    });

    await crawler.run();
});
復(fù)制代碼

// child.js
function computedTotal(arr, cb) {
    // 耗時(shí)計(jì)算任務(wù)
}

// 與主進(jìn)程通信
// 監(jiān)聽主進(jìn)程信號(hào)
process.on('message', (msg) => {
  computedTotal(bigDataArr, (flag) => {
    // 向主進(jìn)程發(fā)送完成信號(hào)
    process.send(flag);
  })
});

// main.js
const { fork } = require('child_process');

app.use(async (ctx, next) => {
  if(ctx.url === '/fetch') {
    const data = ctx.request.body;
    // 通知子進(jìn)程開始執(zhí)行任務(wù),并傳入數(shù)據(jù)
    const res = await createPromisefork('./child.js', data)
  }
  
  // 創(chuàng)建異步線程
  function createPromisefork(childUrl, data) {
    // 加載子進(jìn)程
    const res = fork(childUrl)
    // 通知子進(jìn)程開始work
    data && res.send(data)
    return new Promise(reslove => {
        res.on('message', f => {
            reslove(f)
        })
    })
  }
  
  await next()
})
復(fù)制代碼

// 異步隊(duì)列
const queue = []
// 最大并發(fā)數(shù)
const max_parallel = 6
// 開始指針
let start = 0

for(let i = 0; i < urls.length; i++) {
  // 添加異步隊(duì)列
  queue.push(fetchPage(browser, i, urls[i]))
  if(i &&
      (i+1) % max_parallel === 0 
        || i === (urls.length - 1)) {
    // 每隔6條執(zhí)行一次, 實(shí)現(xiàn)異步分流執(zhí)行, 控制并發(fā)數(shù)
    await Promise.all(queue.slice(start, i+1))
    start = i
  }
}
復(fù)制代碼

// 滾動(dòng)高度
let scrollStep = 1080;
// 最大滾動(dòng)高度, 防止無(wú)限加載的頁(yè)面導(dǎo)致長(zhǎng)效耗時(shí)任務(wù)
let max_height = 30000;
let m = {prevScroll: -1, curScroll: 0}

while (m.prevScroll !== m.curScroll && m.curScroll < max_height) {
    // 如果上一次滾動(dòng)和本次滾動(dòng)高度一樣, 或者滾動(dòng)高度大于設(shè)置的最高高度, 則停止截取
    m = await page.evaluate((scrollStep) => {
      if (document.scrollingElement) {
        let prevScroll = document.scrollingElement.scrollTop;
        document.scrollingElement.scrollTop = prevScroll + scrollStep;
        let curScroll = document.scrollingElement.scrollTop
        return {prevScroll, curScroll}
      }
    }, scrollStep);
    
    // 等待3秒后繼續(xù)滾動(dòng)頁(yè)面, 為了讓頁(yè)面加載充分
    await sleep(3000);
}
// 其他業(yè)務(wù)代碼...
// 截取網(wǎng)頁(yè)快照,并設(shè)置圖片質(zhì)量和保存路徑
const screenshot = await page.screenshot({path: `static/${uid}.jpg`, fullPage: true, quality: 70});
復(fù)制代碼

const txt = await page.$eval('body', el => {
    // el即為dom節(jié)點(diǎn), 可以對(duì)body的子節(jié)點(diǎn)進(jìn)行提取,分析
    return {...}
})
復(fù)制代碼

const Koa  = require('koa');
const { resolve } = require('path');
const staticServer = require('koa-static');
const koaBody = require('koa-body');
const cors = require('koa2-cors');
const logger = require('koa-logger');
const glob = require('glob');
const { fork } = require('child_process');

const app = new Koa();
// 創(chuàng)建靜態(tài)目錄
app.use(staticServer(resolve(__dirname, './static')));
app.use(staticServer(resolve(__dirname, './db')));
app.use(koaBody());
app.use(logger());

const config = {
  imgPath: resolve('./', 'static'),
  txtPath: resolve('./', 'db')
}

// 設(shè)置跨域
app.use(cors({
  origin: function (ctx) {
      if (ctx.url.indexOf('fetch') > -1) {
        return '*'; // 允許來(lái)自所有域名請(qǐng)求
      }
      return ''; // 這樣就能只允許 http://localhost 這個(gè)域名的請(qǐng)求了
  },
  exposeHeaders: ['WWW-Authenticate', 'Server-Authorization'],
  maxAge: 5,  //  該字段可選，用來(lái)指定本次預(yù)檢請(qǐng)求的有效期，單位為秒
  credentials: true,
  allowMethods: ['GET', 'POST', 'PUT', 'DELETE'],
  allowHeaders: ['Content-Type', 'Authorization', 'Accept', 'x-requested-with'],
}))

// 創(chuàng)建異步線程
function createPromisefork(childUrl, data) {
  const res = fork(childUrl)
    data && res.send(data)
    return new Promise(reslove => {
      res.on('message', f => {
        reslove(f)
      })
    })
}

app.use(async (ctx, next) => {
  if(ctx.url === '/fetch') {
    const data = ctx.request.body;
    const res = await createPromisefork('./child.js', data)
    // 獲取文件路徑
    const txtUrls = [];
    let reg = /.*?(\d+)\.\w*$/;
    glob.sync(`${config.txtPath}/*.*`).forEach(item => {
      if(reg.test(item)) {
        txtUrls.push(item.replace(reg, '$1'))
      }
    })

    ctx.body = {
      state: res,
      data: txtUrls,
      msg: res ? '抓取完成' : '抓取失敗,原因可能是非法的url或者請(qǐng)求超時(shí)或者服務(wù)器內(nèi)部錯(cuò)誤'
    }
  }
  await next()
})

app.listen(80)
復(fù)制代碼