Python USES scrapy to grab information from the website sitemap
- 2020-05-07 19:57:37
- OfStack
This article illustrates an example of how Python USES scrapy to grab information from a website sitemap. Share with you for your reference. The details are as follows:
import re
from scrapy.spider import BaseSpider
from scrapy import log
from scrapy.utils.response import body_or_str
from scrapy.http import Request
from scrapy.selector import HtmlXPathSelector
class SitemapSpider(BaseSpider):
name = "SitemapSpider"
start_urls = ["http://www.domain.com/sitemap.xml"]
def parse(self, response):
nodename = 'loc'
text = body_or_str(response)
r = re.compile(r"(<%s[\s>])(.*?)(</%s>)"%(nodename,nodename),re.DOTALL)
for match in r.finditer(text):
url = match.group(2)
yield Request(url, callback=self.parse_page)
def parse_page(self, response):
hxs = HtmlXPathSelector(response)
#Mock Item
blah = Item()
#Do all your page parsing and selecting the elemtents you want
blash.divText = hxs.select('//div/text()').extract()[0]
yield blah
I hope this article has been helpful to your Python programming.