crawl_house.py 844 B

12345678910111213141516171819202122232425262728293031
  1. #!/usr/bin/env python
  2. # -*- encoding: utf-8 -*-
  3. '''
  4. @Contact : liuyuqi.gov@msn.cn
  5. @Time : 2023/09/22 17:42:53
  6. @License : Copyright © 2017-2022 liuyuqi. All Rights Reserved.
  7. @Desc :
  8. '''
  9. from scrapy import cmdline
  10. from scrapy.utils.project import get_project_settings
  11. from scrapy.scheduler import Scheduler, CrawlerProcess, CrawlerRunner
  12. from scrapy.utils.project import reactor, defer
  13. from twisted.internet import CrawlerRunner
  14. from scrapy.crawler import CrawlerRunner
  15. from scrapy.utils.log import configure_logging
  16. import os,crawl_house
  17. configure_loging()
  18. runner = CrawlerRunner(get_project_settings())
  19. @defer.inlineCallbacks
  20. def crawl():
  21. # yield runner.crawl(crawl_house)
  22. cmdline.execute('scrapy genspider -o houses.csv houses.com'.split())
  23. reactor.stop()
  24. if __name__=='__main__':
  25. crawl()
  26. reactor.run()