settings.py 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129
  1. # -*- coding: utf-8 -*-
  2. # Scrapy settings for elabSpider project
  3. #
  4. # For simplicity, this file contains only settings considered important or
  5. # commonly used. You can find more settings consulting the documentation:
  6. #
  7. # https://doc.scrapy.org/en/latest/topics/settings.html
  8. # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
  9. # https://doc.scrapy.org/en/latest/topics/spider-middleware.html
  10. BOT_NAME = 'elabSpider'
  11. SPIDER_MODULES = ['elabSpider.spiders']
  12. NEWSPIDER_MODULE = 'elabSpider.spiders'
  13. # MONGO_HOST = '139.196.5.59' #测试环境
  14. # MONGO_HOST = '139.196.108.59' #正式环境,已废弃
  15. MONGO_HOST = 'mongodb://logdb:logdb@dds-uf6da0fedc9881d41450-pub.mongodb.rds.aliyuncs.com:3717,dds-uf6da0fedc9881d42459-pub.mongodb.rds.aliyuncs.com:3717/logdb?replicaSet=mgset-12835903'
  16. MONGO_PORT = 27017
  17. MONGO_DB = 'logdb'
  18. MONGO_COLL = 'ershoufang'
  19. MONGO_USER = 'dbuser'
  20. MONGO_PSW = 'elab@123'
  21. PROXY_HOST = 'http://http-dyn.abuyun.com'
  22. PROXY_PORT = '9020'
  23. # PROXY_IDENTIFY = 'HY39548V0FZ45UKD'
  24. # PROXY_SECRETKEY = '07DBA6C5E470150B'
  25. USER_AGENT_PATH = 'fake_useragent.json'
  26. # LOG_FILE = 'spider.log'
  27. # Crawl responsibly by identifying yourself (and your website) on the user-agent
  28. #USER_AGENT = 'elabSpider (+http://www.yourdomain.com)'
  29. # Obey robots.txt rules
  30. ROBOTSTXT_OBEY = False
  31. # Configure maximum concurrent requests performed by Scrapy (default: 16)
  32. #CONCURRENT_REQUESTS = 32
  33. # Configure a delay for requests for the same website (default: 0)
  34. # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
  35. # See also autothrottle settings and docs
  36. #DOWNLOAD_DELAY = 3
  37. # The download delay setting will honor only one of:
  38. #CONCURRENT_REQUESTS_PER_DOMAIN = 16
  39. CONCURRENT_REQUESTS_PER_IP = 5
  40. # Disable cookies (enabled by default)
  41. COOKIES_ENABLED = False
  42. # COOKIES_DEBUG = True
  43. # Disable Telnet Console (enabled by default)
  44. #TELNETCONSOLE_ENABLED = False
  45. # Override the default request headers:
  46. #DEFAULT_REQUEST_HEADERS = {
  47. # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  48. # 'Accept-Language': 'en',
  49. #}
  50. # Enable or disable spider middlewares
  51. # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
  52. #SPIDER_MIDDLEWARES = {
  53. # 'elabSpider.middlewares.ElabspiderSpiderMiddleware': 543,
  54. #}
  55. # Enable or disable downloader middlewares
  56. # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
  57. DOWNLOADER_MIDDLEWARES = {
  58. 'elabSpider.middlewares.UserAgent': 1,
  59. # 'elabSpider.middlewares.TooManyRequestsRetryMiddleware': 500,
  60. 'elabSpider.middlewares.ElabspiderDownloaderMiddleware': 543,
  61. 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 550,
  62. 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
  63. 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': None,
  64. 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
  65. # 'scrapy.downloadermiddlewares.retry.RetryMiddleware': None
  66. }
  67. # Enable or disable extensions
  68. # See https://doc.scrapy.org/en/latest/topics/extensions.html
  69. #EXTENSIONS = {
  70. # 'scrapy.extensions.telnet.TelnetConsole': None,
  71. #}
  72. # Configure item pipelines
  73. # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
  74. ITEM_PIPELINES = {
  75. 'elabSpider.pipelines.ElabspiderPipeline': 300,
  76. }
  77. # Enable and configure the AutoThrottle extension (disabled by default)
  78. # See https://doc.scrapy.org/en/latest/topics/autothrottle.html
  79. # AUTOTHROTTLE_ENABLED = True
  80. # The initial download delay
  81. # AUTOTHROTTLE_START_DELAY = 0.25
  82. # The maximum download delay to be set in case of high latencies
  83. # AUTOTHROTTLE_MAX_DELAY = 60
  84. # The average number of requests Scrapy should be sending in parallel to
  85. # each remote server
  86. # AUTOTHROTTLE_TARGET_CONCURRENCY = 5.0
  87. # Enable showing throttling stats for every response received:
  88. # AUTOTHROTTLE_DEBUG = True
  89. # Enable and configure HTTP caching (disabled by default)
  90. # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
  91. #HTTPCACHE_ENABLED = True
  92. #HTTPCACHE_EXPIRATION_SECS = 0
  93. #HTTPCACHE_DIR = 'httpcache'
  94. #HTTPCACHE_IGNORE_HTTP_CODES = []
  95. #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
  96. REDIRECT_ENABLED = False
  97. RETRY_ENABLED = True
  98. RETRY_TIMES = 10
  99. RETRY_HTTP_CODES = [403, 429, 404, 301, 302, 503]
  100. HTTPERROR_ALLOWED_CODES = [403, 429, 404, 301, 302, 503]
  101. DOWNLOAD_TIMEOUT = 15
  102. # RANDOMIZE_DOWNLOAD_DELAY = False
  103. # CONCURRENT_REQUESTS_PER_IP = 40