问题描述

做了一个小说爬虫，能获取到数据，但是无法获取全部的章节，会有章节缺失，并且爬到一定书籍之后程序就不在卡主不在运行，

问题出现的环境背景及自己尝试过哪些方法

相关代码

// 请把代码文本粘贴到下方（请勿用图片代替代码）

class Chapter{
    constructor(){

    }
    // 解析每一章的书籍信息内容
    async getChapter(url,obj){
        
        try{
            obj = obj || {}
            let chapterId = url.substring(url.lastIndexOf('/')+1,url.lastIndexOf('.'))
            let bookId = url.substring(0,url.lastIndexOf('/')).substring(url.substring(0,url.lastIndexOf('/')).lastIndexOf('/')+1)
            let databaseChapter = await modelBook.Chapter.find({chapter_id: chapterId})
            

            let chapter = await superagent.get(url)
            let $ = cheerio.load(chapter.text)
            let newChapter = {
                title: $('.bookname h1').text(),
                content: $('#content').text(),
                chapter_id: chapterId,
                book_id: bookId,
                name: obj.name || '',
                author: obj.author || '',
            }
            let newChapterModel = new modelBook.Chapter(newChapter)
            if(databaseChapter.length > 0){
                // console.log('数据已存在')
                await modelBook.Chapter.update({chapter_id: chapterId},newChapter)
                return
            }
            newChapterModel.save()
            console.log('数据保存成功',chapterId)
        }catch(err){
            console.log('爬虫失败',url)
            throw Error(err)
        }
        
    }
    
}

class Book extends Chapter{
    constructor(){
        super()
        this.allcount = 0
    }

    // 根据书籍地址获取书籍的章节列表信息(并处理书籍的信息保存到数据库中)
    async getChapterList(url){
        
        try{
            let result = await superagent.get(url)
            let $ = cheerio.load(result.text)
            let book_id = url.substring(0,url.lastIndexOf('/'))
            book_id = book_id.substring(book_id.lastIndexOf('/')+1)
            let author = $('#info p').eq(0).text().substring($('#info p').eq(0).text().lastIndexOf('：')+1)
            let newBook = {
                name: $('#info h1').text(),
                author: author,
                introduction: $('#intro p').eq(1).text(),
                book_id: book_id,
                img_url: $('#fmimg img').attr('src'),
                tag: $('.con_top a').eq(2).text()
            }
            let newBookModel = new modelBook.Book(newBook)
            let bookArr = await modelBook.Book.find({book_id: book_id})
            let chapterCount = await modelBook.Chapter.find({book_id: book_id})
                                .count()
            
            if(bookArr.length > 0){
                // console.log('书籍信息已经存在 ')
                await modelBook.Book.update({book_id: book_id},newBook)
            }else{
                newBookModel.save()
                console.log('书籍保存成功')
            }
            let oneBookChapterList = []
            $('#list dd a').map(async (index,val) => {
                let href = $(val).attr('href')
                oneBookChapterList.push({
                    url: urlBase + href,
                    data: {
                        name: $('#info h1').text(),
                        author: author,
                    }
                })
                
            })
            if(chapterCount == $('#list dd a').length){
                console.log('书籍信息完整'+book_id)
                return
            }else{
                console.log('开始获取章节书籍信息不完整'+book_id)
                async.mapLimit(oneBookChapterList,3, async (val) => {
                    await this.getChapter(val.url, val.data)
                })
            }
            


        }catch(err){
            console.log('获取文章列表失败')
        }
    }

    init(){
        this.getAll(url)
    }
    async getAll(url){
        console.log(url)
        try{
            let result = await superagent.get(url)
            let $ = cheerio.load(result.text)
            let urls = []
            $('#main .novellist li').map((index, val) => {
                let bookUrl = $(val).find('a').attr('href')
                urls.push(bookUrl)
            })
            console.log(urls.length)
            async.mapLimit(urls,1,async (url) => {
                await this.getChapterList(url)
                this.allcount += 1
                console.log('这是第'+this.allcount + '本书，地址：'+url)
            })
        }catch(err){
            console.log('书籍列表请求失败')
        }
    }
}

你期待的结果是什么？实际看到的错误信息又是什么？