循环只给我第一行（puppeteer js）

Question

我正在尝试获取价格和卖家数据，但我只得到第一行，为什么？ 我正在努力学习js，但是在这里我花了很多时间，请帮助。

我从不同站点获取代码并根据我的要求对其进行修改，但仍然无法正常工作

Puppeteer js、nodejs

var url = 'https://www.amazon.co.uk/gp/offer-listing/3770436385/ref=olp_f_used?ie=UTF8&f_new=true&f_usedAcceptable=true&f_usedGood=true&f_used=true&f_usedLikeNew=true&f_usedVeryGood=true';

const puppeteer = require('puppeteer');

(async function main(){
    try{

        const browser = await puppeteer.launch({headless: false });
        const page = await browser.newPage();
        page.setUserAgent('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/67.0.3372.0 Safari/537.36');

        await page.goto(url);
        await page.waitForSelector('div.navFooterLine:nth-child(6)');

        const sections = await page.$$('.olpOffer');

        console.log(sections.length);

        for (const section of sections){

            const result = await page.evaluate(() => {
                //  let amaOffer = document.querySelector('#olpOfferList img').innerText;
                //  let amaPrice = document.querySelector('#olpOfferList img').innerText;
                let seller = document.querySelector('.a-text-bold a').innerText;
                let price = document.querySelector('.olpOfferPrice').innerText;

                return {
                //  amaOffer,
                    seller,
                    price

            }

        } )
        console.log(result);
    }

    } catch (e) {
        console.log('Our  error', e);
    }


})();

Answer 1

你实际上是在这样做：

for (const section of sections) {
   const result = someValue;
   console.log(result);
}

因此，您只是在循环中的 over 上声明了一个新的 result 变量，而不是将每个单独的结果累积到一个数组中，这样您就可以获得所有结果。

不是 100% 清楚您想要结果的位置，但您至少可以将它们累积到一个数组中，如下所示：

(async function main() {
    try {

        const browser = await puppeteer.launch({headless: false});
        const page = await browser.newPage();
        page.setUserAgent('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3372.0 Safari/537.36');
        await page.goto(url);
        await page.waitForSelector('div.navFooterLine:nth-child(6)');
        const sections = await page.$$('.olpOffer');

        console.log(sections.length);

        let results = [];

        for (const section of sections) {

            const result = await page.evaluate(() => {
                //  let amaOffer = document.querySelector('#olpOfferList img').innerText;
                //  let amaPrice = document.querySelector('#olpOfferList img').innerText;
                let seller = document.querySelector('.a-text-bold a').innerText;
                let price = document.querySelector('.olpOfferPrice').innerText;

                return {
                    //  amaOffer,
                    seller,
                    price

                }

            });
            console.log(result);
            results.push(result);
        }
        // see all results here
        console.log(results);
        return results;

    } catch (e) {
        console.log('Our  error', e);
    }
})();

Answer 2

我知道问题出在哪里。

1. Puppeteer JS 在两个引擎中运行 javascript，两者的作用域不同，所以我们需要将 section 数字传递给 page.evaluate
2. 亚马逊网站显示的结果是亚马逊自己的产品，选择器没有选择正确的，所以我把它改成了.olpSellerName
3. 我添加了一些改进以使脚本运行得更快。

希望这能有所帮助。

let url = 'https://www.amazon.co.uk/gp/offer-listing/3770436385/ref=olp_f_used?ie=UTF8&f_new=true&f_usedAcceptable=true&f_usedGood=true&f_used=true&f_usedLikeNew=true&f_usedVeryGood=true'
let chromeUA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3372.0 Safari/537.36'

const puppeteer = require('puppeteer')

;(async function main() {
    try {

        const browser = await puppeteer.launch({headless: false})
        const page = (await browser.pages())[0]
        page.setUserAgent(chromeUA)
        page.setDefaultNavigationTimeout(0)
        page.setRequestInterception(true)

        page.on('request', async request => {
            if  (
                    request.resourceType() === 'image' ||
                    request.resourceType() === 'media' ||
                    request.resourceType() === 'script' ||
                    request.resourceType() === 'stylesheet' ||
                    request.resourceType() === 'font' ||
                    request.resourceType() === 'other'
                ) {
                request.abort()
            } else {
                request.continue()
            }
        })

        await page.goto(url)
        await page.waitForSelector('div.navFooterLine:nth-child(6)')
        const sections = await page.$$('.olpOffer')

        let results = []

        for (let num = 0; num < sections.length; num++) {
            let result = await page.evaluate( (num) => {
                let seller = document.querySelectorAll('.olpSellerName')[num].innerText
                let price = document.querySelectorAll('.olpOfferPrice')[num].innerText

                return {
                    //  amaOffer,
                    seller,
                    price
                }

            }, num)
            results.push(result)
        }
        // see all results here
        console.log(results)
        return results

    } catch (e) {
        console.log('Our  error', e);
    }
})()

---

我还有一个问题：

我想循环访问不同的亚马逊网站

let AmazonSites = ['de','co.uk','ca'];
for(AmazonSite of AmazonSites){

let url = 'https://www.amazon.'+ AmazonSite +'/gp/offer-listing/'+ Asin +'/ref=olp_f_used?ie=UTF8&f_new=true&f_usedAcceptable=true&f_usedGood=true&f_used=true&f_usedLikeNew=true&f_usedVeryGood=true';


await page.goto(url)

所以你可以通过for循环重复脚本来发出多个请求

let domains = ['co.uk', 'de', 'ca']
let url = 'https://www.amazon.tld/gp/offer-listing/3770436385/ref=olp_f_used?ie=UTF8&f_new=true&f_usedAcceptable=true&f_usedGood=true&f_used=true&f_usedLikeNew=true&f_usedVeryGood=true'
let chromeUA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3372.0 Safari/537.36'

const puppeteer = require('puppeteer')

;(async function main() {
    try {

        const browser = await puppeteer.launch({headless: false})
        const page = (await browser.pages())[0]
        page.setUserAgent(chromeUA)
        page.setDefaultNavigationTimeout(0)
        page.setRequestInterception(true)

        page.on('request', async request => {
            if  ( request.resourceType() === 'document' ) {
                request.continue()
            } else {
                request.abort()
            }
        })

        for (let tld in domains) {
            await page.goto(url.replace('tld', domains[tld]))
            await page.waitForSelector('div.navFooterLine:nth-child(6)')
            const sections = await page.$$('.olpOffer')

            let results = []

            for (let num = 0; num < sections.length; num++) {
                let result = await page.evaluate( (num) => {
                    let seller = document.querySelectorAll('.olpSellerName')[num].innerText
                    let price = document.querySelectorAll('.olpOfferPrice')[num].innerText

                    return {
                        //  amaOffer,
                        seller,
                        price
                    }

                }, num)
                results.push(result)
            }
            // see all results here
            console.log (`\n\nRESULT FOR DOMAIN TLD : www.amazon.${domains[tld]}`)
            console.log(results)
        }

        await browser.close()

    } catch (e) {
        console.log('Our  error', e);
    }
})()