Python to grab jingdong mall mobile phone list url instance code

  • 2020-04-02 13:17:17
  • OfStack


#-*- coding: UTF-8 -*-
'''
Created on 2013-12-5
@author: good-temper
'''
import urllib2
import bs4
import time
def getPage(urlStr):
    '''
                 Get the page content 
    '''
    content = urllib2.urlopen(urlStr).read()
    return content
def getNextPageUrl(currPageNum):
    #http://list.jd.com/9987-653-655-0-0-0-0-0-0-0-1-1- The page number -1-1-72-4137-33.html
    url =  u'http://list.jd.com/9987-653-655-0-0-0-0-0-0-0-1-1-'+str(currPageNum+1)+'-1-1-72-4137-33.html'

    # Is there a next page 
    content = getPage(url);
    soup = bs4.BeautifulSoup(content)
    list = soup.findAll('span',{'class':'next-disabled'});
    if(len(list) == 0):
        return url
    return ''

def analyzeList():
    pageNum = 0
    list = []
    url = getNextPageUrl(pageNum)
    while url !='':
        soup = bs4.BeautifulSoup(getPage(url))
        pagelist = soup.findAll('div',{'class':'p-name'})
        for elem in pagelist:
            soup1 =  bs4.BeautifulSoup(str(elem))
            list.append(soup1.find('a')['href'])

        pageNum = pageNum+1
        print pageNum
        url = getNextPageUrl(pageNum)
    return list
def analyzeContent(url):

    return ''
def writeToFile(list, path):
    f = open(path, 'a')
    for elem in list:
        f.write(elem+'n')
    f.close()
if __name__ == '__main__':
    list = analyzeList()
    print ' A total of fetching '+str(len(list))+' article n'

    writeToFile(list, u'E:\jd_phone_list.dat');


Related articles: