v2ex.py

from acrawler import Crawler, Request, register


class V2EXCrawler(Crawler):
    def start_requests(self):
        yield Request(
            url="https://www.v2ex.com/?tab=hot",
            callback=self.parse_hot,
            recrawl=5,
            links_to_abs=True,
        )

    def parse(self, response):
        print(
            "This is default callback function! Auto-combined to any request generated by start_requests()."
        )

    def parse_hot(self, response):
        aa = response.sel.css(".item_title a")
        for a in aa:
            d = {
                "url": response.urljoin(a).split("#")[0],
                "title": a.css("::text").get(),
            }
            yield d


@register(family="DefaultItem")
def process_d(d):
    print(d.content)


if __name__ == "__main__":
    V2EXCrawler().run()