first

README.md

@@ -0,0 +1,37 @@
+# abuyunRecharge
+
+#### 项目介绍
+阿布云自动续费
+
+#### 软件架构
+软件架构说明
+
+
+#### 安装教程
+
+1. xxxx
+2. xxxx
+3. xxxx
+
+#### 使用说明
+
+1. xxxx
+2. xxxx
+3. xxxx
+
+#### 参与贡献
+
+1. Fork 本项目
+2. 新建 Feat_xxx 分支
+3. 提交代码
+4. 新建 Pull Request
+
+
+#### 码云特技
+
+1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md
+2. 码云官方博客 [blog.gitee.com](https://blog.gitee.com)
+3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解码云上的优秀开源项目
+4. [GVP](https://gitee.com/gvp) 全称是码云最有价值开源项目，是码云综合评定出的优秀开源项目
+5. 码云官方提供的使用手册 [http://git.mydoc.io/](http://git.mydoc.io/)
+6. 码云封面人物是一档用来展示码云会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/)

abuyun_renew.py

@@ -0,0 +1,138 @@
+# -*- coding:utf-8 -*-
+
+# @Time    : 2018/5/14 3:58 PM
+
+# @Author  : Swing
+
+
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+import launch_spider
+import email_util
+import traceback
+
+browser = webdriver.Chrome()
+# browser.maximize_window()
+
+timeout = 180
+time_interval = 0.1
+
+
+def open_chrome():
+    browser.get('https://center.abuyun.com/#/cloud/http-proxy/tunnel/lists')
+
+
+def recharge(is_launch_spider=False):
+    # 等待网页加载完成
+    browser.get('https://center.abuyun.com/#/cloud/http-proxy/tunnel/lists')
+    wait_preloader()
+
+    if browser.title == '阿布云 - 服务管理控制台':
+
+        element_xpath = r'//a[text()="动态版"]'
+        element = WebDriverWait(browser, timeout, time_interval).until(EC.element_to_be_clickable((By.XPATH, element_xpath)))
+        element.click()
+        # 选中要续费的代理渠道
+        checkbox_xpath = r'//tbody/tr/td/div[1]/label/span'
+        # WebDriverWait(browser, timeout, 1).until(EC.presence_of_element_located((By.XPATH, checkbox_xpath)))
+        checkbox = WebDriverWait(browser, timeout, time_interval).until(EC.element_to_be_clickable((By.XPATH, checkbox_xpath)))
+        checkbox.click()
+
+        # 点击续费按钮
+        try:
+            WebDriverWait(browser, timeout, time_interval).until(
+                lambda x: x.find_element_by_xpath(r'//tbody/tr/td/div[1]/label/input[@type="checkbox"]').is_selected())
+        except TimeoutException:
+            email_util.send_email('Abuyun renew timeout', traceback.format_exc())
+        except:
+            email_util.send_email('Abuyun renew error', traceback.format_exc())
+
+        renew_xpath = r'//div[@class="panel-footer"]/div[@class="row"]/div/button[text()=" 续费"][not(@disabled)]'
+        # WebDriverWait(browser, timeout, 1).until(EC.presence_of_element_located((By.XPATH, renew_xpath)))
+        renew = WebDriverWait(browser, timeout, time_interval).until(EC.element_to_be_clickable((By.XPATH, renew_xpath)))
+        renew.click()
+
+        WebDriverWait(browser, timeout, time_interval).until(EC.visibility_of_element_located((By.XPATH, r'//div[@class="modal-dialog modal-mlg"]')))
+
+        # 选择续费周期
+        unit_select_xpath = r'//div[@class="input-group unit-input-group"]/select'
+        # WebDriverWait(browser, timeout, 1).until(EC.presence_of_element_located((By.XPATH, unit_select_xpath)))
+        unit_select = WebDriverWait(browser, timeout, time_interval).until(EC.element_to_be_clickable((By.XPATH, unit_select_xpath)))
+
+        select = Select(unit_select)
+        select.select_by_visible_text('时')
+
+        number = WebDriverWait(browser, timeout, time_interval).until(lambda x: x.find_element_by_xpath(r'//input[@type="number"]'))
+        number.clear()
+        number.send_keys('1')
+
+        charge_button = browser.find_element_by_xpath(r'//div[@class="modal-footer"]/button[text()=" 续费"]')
+        charge_button.click()
+
+        # 等待充值页面加载成功
+        WebDriverWait(browser, timeout, time_interval).until(EC.url_contains('https://center.abuyun.com/#/trade/order/confirm?'))
+        WebDriverWait(browser, timeout, time_interval).until(EC.invisibility_of_element_located((By.XPATH, r'//div[contains(@class, "modal")]')))
+        wait_preloader()
+
+        balance_xpath = r'//td[span/span="可用余额"]/strong'
+        WebDriverWait(browser, timeout, time_interval).until(EC.visibility_of_element_located((By.XPATH, balance_xpath)))
+        WebDriverWait(browser, timeout, time_interval).until(EC.invisibility_of_element_located((By.XPATH, r'//div[contains(@class, "modal-backdrop")]')))
+        WebDriverWait(browser, timeout, time_interval).until_not(EC.presence_of_element_located((By.XPATH, r'//div[contains(@class, "modal-backdrop")]')))
+
+        if '钱包余额不足, 请先充值或直接使用支付宝支付' in browser.page_source:
+            email_util.send_email('Abuyun renew error', 'Lack of balance')
+            print('Lack of balance')
+            return None
+
+        use_wallet_xpath = r'//table[@class="table table-bordered ng-scope"]/tbody/tr/td[3]/label/span'
+        use_wallet = WebDriverWait(browser, timeout, time_interval).until(EC.element_to_be_clickable((By.XPATH, use_wallet_xpath)))
+        use_wallet.click()
+
+        try:
+            balance_string = browser.find_element_by_xpath(balance_xpath).text
+            balance = float(balance_string)
+            if balance < 40:
+                email_util.send_email('Abuyun balance alert', 'Balance: ' + balance_string + ' Please recharge timely!')
+        except:
+            email_util.send_email('Abuyun balance check error', traceback.format_exc())
+
+        money = WebDriverWait(browser, timeout, time_interval).until(EC.visibility_of_element_located((By.XPATH, r'//span[@class="text-money"]')))
+        if money.text == '0.00':
+            print('price 0')
+            pay = WebDriverWait(browser, timeout, time_interval).until(EC.element_to_be_clickable((By.XPATH, r'//button[text()="马上支付"]')))
+            pay.click()
+        else:
+            email_util.send_email('Abuyun renew error', 'Price not 0')
+            return None
+
+        # 等待订单列表加载成功
+        WebDriverWait(browser, timeout, time_interval).until(EC.url_contains(r'https://center.abuyun.com/#/trade/order/lists?'))
+        wait_preloader()
+        WebDriverWait(browser, timeout, time_interval).until(
+            EC.invisibility_of_element_located((By.XPATH, r'//div[contains(@class, "modal")]')))
+
+        WebDriverWait(browser, timeout, time_interval).until(EC.visibility_of_element_located((By.XPATH, r'//div[contains(@class, "sweet-alert")]')))
+        confirm_button = WebDriverWait(browser, timeout, time_interval).until(
+            EC.element_to_be_clickable((By.XPATH, r'//button[@class="confirm"]')))
+        if '已从钱包扣款，订单支付完成' in browser.page_source:
+            print('Renew completed')
+            if is_launch_spider:
+                launch_spider.start_spider()
+            confirm_button.click()
+
+        else:
+            email_util.send_email('Abuyun renew error', 'Abuyun renew error')
+            print('Renew failed')
+
+    else:
+        email_util.send_email('Abuyun Login failed', 'Abuyun Login failed')
+        print('Login failed')
+
+
+def wait_preloader():
+    WebDriverWait(browser, timeout, time_interval).until(
+        EC.invisibility_of_element_located((By.XPATH, r'//div[contains(@class, "preloader")]')))

abuyun_server.py

@@ -0,0 +1,98 @@
+#!/usr/local/bin/python3
+
+# -*- coding:utf-8 -*-
+
+# @Time    : 2018/5/18 3:18 PM
+
+# @Author  : Swing
+
+
+from tornado.web import RequestHandler
+import abuyun_renew
+import statistics
+import tornado.ioloop
+import email_util
+import traceback
+import json
+import loop_task
+
+
+class MainHandler(RequestHandler):
+    def get(self, *args, **kwargs):
+        self.write('Hello world')
+
+    def post(self):
+        self.write('Hello world')
+
+
+class OpenChrome(RequestHandler):
+    def get(self, *args, **kwargs):
+        abuyun_renew.open_chrome()
+        self.write('Hello world')
+
+    def post(self):
+        self.write('Hello world')
+
+
+class ReCharge(RequestHandler):
+    def get(self, *args, **kwargs):
+        try:
+            abuyun_renew.recharge(is_launch_spider=True)
+            loop_task.loop_start()
+            self.finish(Response().to_json())
+        except:
+            email_util.send_email('Abuyun renew error', traceback.format_exc())
+            print('Daily statistic err! reason: ' + traceback.format_exc())
+            self.finish(Response(False, traceback.format_exc()).to_json())
+
+
+class DailyStatistic(RequestHandler):
+    def get(self, *args, **kwargs):
+        try:
+            statistics.daily_statistic()
+            self.finish(Response().to_json())
+        except:
+            email_util.send_email('Daily statistic err', traceback.format_exc())
+            print('Daily statistic err! reason: ' + traceback.format_exc())
+            self.finish(Response(False, traceback.format_exc()).to_json())
+
+
+class LoopTaskStart(RequestHandler):
+    def get(self, *args, **kwargs):
+        try:
+            loop_task.loop_start()
+            self.finish(Response().to_json())
+        except:
+            print(traceback.format_exc())
+
+
+class LoopTaskStop(RequestHandler):
+    def get(self, *args, **kwargs):
+        try:
+            loop_task.loop_stop()
+            self.finish(Response().to_json())
+        except:
+            print(traceback.format_exc())
+
+
+class Response(object):
+    def __init__(self, success=True, message='ok'):
+        self.success = success
+        self.message = message
+
+    def to_json(self):
+        return json.dumps(self, default=lambda obj: obj.__dict__, sort_keys=True, indent=4)
+
+
+application = tornado.web.Application([
+    (r"/openChrome", OpenChrome),
+    (r'/reCharge', ReCharge),
+    (r'/statistic', DailyStatistic),
+    (r'/loopStart', LoopTaskStart),
+    (r'/loopStop', LoopTaskStop),
+])
+
+if __name__ == "__main__":
+    application.listen(8888)
+    # print('Tornado start at port: 8888')
+    tornado.ioloop.IOLoop.instance().start()

email_util.py

@@ -0,0 +1,40 @@
+# -*- coding:utf-8 -*-
+
+# @Time    : 2018/5/23 11:49 AM
+
+# @Author  : Swing
+
+
+import smtplib
+from email.mime.text import MIMEText
+from email.header import Header
+import traceback
+
+mail_host = 'smtp.exmail.qq.com'
+mail_user = 'zhaojh@elab-plus.com'
+mail_pass = 'Elab@123'
+
+sender = 'zhaojh@elab-plus.com'
+receivers = ['zhaojh@elab-plus.com']
+
+
+def send_email(title, content, receiver=None):
+    message = MIMEText(content, 'plain')
+    message['From'] = Header("Tornado service")
+    message['To'] = Header("Admin")
+    message['subject'] = Header(title)
+
+    try:
+        smtp_obj = smtplib.SMTP()
+        smtp_obj.connect(mail_host, 25)
+        smtp_obj.login(mail_user, mail_pass)
+        if receiver:
+            smtp_obj.sendmail(sender, receiver, message.as_string())
+        else:
+            smtp_obj.sendmail(sender, receivers, message.as_string())
+        print('Mail sent successfully')
+    except smtplib.SMTPException:
+        print('Error: Mail send failed' + traceback.format_exc())
+
+# 测试代码
+# send_email("我是测试标题", "我是测试内容")

launch_spider.py

@@ -0,0 +1,23 @@
+# -*- coding:utf-8 -*-
+
+# @Time    : 2018/5/16 1:35 PM
+
+# @Author  : Swing
+
+
+import requests
+
+
+def start_spider():
+    url = 'http://192.168.4.246:8080/schedule.json'
+
+    project_name = 'default'
+    spider_name_list = ['lfsSoldAverage', 'lfsrentalHouse', 'lfsresoldHouse', 'rentalHouse', 'nbresoldHouse', 'sjkresoldHouse', 'sjkrentalHouse', 'ftxrentalHouse']
+
+    for spider_name in spider_name_list:
+        post_data = {'project': project_name, 'spider': spider_name}
+        result = requests.post(url, data=post_data)
+        print(result.text)
+
+
+# start_spider()

loop_task.py

@@ -0,0 +1,40 @@
+# -*- coding:utf-8 -*-
+
+# @Time    : 2018/6/20 11:47 AM
+
+# @Author  : Swing
+
+
+from tornado import ioloop
+import scrapyd_manager
+import statistics
+import abuyun_renew
+
+time_interval = 1000 * 60 * 55
+# time_interval = 1000 * 3
+
+
+def loop_handle():
+    # print('loop\n')
+    running_list = scrapyd_manager.get_spider_list()
+    if running_list and len(running_list) > 0:
+        abuyun_renew.recharge()
+        # pass
+    else:
+        statistics.daily_statistic()
+        loop_stop()
+        # print('loop stop')
+
+
+task = ioloop.PeriodicCallback(loop_handle, time_interval)
+
+
+def loop_start():
+    task.start()
+
+
+def loop_stop():
+    task.stop()
+
+
+# loop_start()

scrapyd_manager.py

@@ -0,0 +1,28 @@
+# -*- coding:utf-8 -*-
+
+# @Time    : 2018/6/20 11:55 AM
+
+# @Author  : Swing
+
+
+import requests
+import email_util
+import traceback
+
+
+def get_spider_list():
+    try:
+        url = r'http://192.168.4.246:8080/listjobs.json?project=default'
+        result = requests.get(url).json()
+
+        if result['status'] == 'ok':
+            running_list = result['running']
+            return running_list
+        else:
+            email_util.send_email('Scrapyd status check error', result)
+            return None
+    except:
+        email_util.send_email('Scrapyd status check error', traceback.format_exc())
+        return None
+
+# get_spider_list()

statistics.py

@@ -0,0 +1,240 @@
+# -*- coding:utf-8 -*-
+
+# @Time    : 2018/5/22 1:25 PM
+
+# @Author  : Swing
+
+import pymongo
+import time
+import re
+import email_util
+
+MONGO_HOST = 'mongodb://logdb:logdb@dds-uf6da0fedc9881d41450-pub.mongodb.rds.aliyuncs.com:3717,dds-uf6da0fedc9881d42459-pub.mongodb.rds.aliyuncs.com:3717/logdb?replicaSet=mgset-12835903'
+
+def daily_statistic():
+
+    now_date: time = time.localtime(time.time())
+    year = now_date.tm_year
+    month = now_date.tm_mon
+    day = now_date.tm_mday
+
+    month_str = str(year) + '-'
+    if month < 10:
+        month_str += '0'
+    month_str += str(month)
+    day_str = month_str + '-'
+    if day < 10:
+        day_str += '0'
+    day_str += str(day)
+
+    date_reg = re.compile(day_str)
+    area_reg = re.compile(r'(^[5][5-9][^\d])|(^[6][0-5][^\d])')
+
+    city_residential_query_args = {'date': date_reg, 'area': area_reg, 'property_type': '普通住宅', 'decoration': {'$in': ['精装修', '豪华装修']}}
+    city_department_query_args = {'date': date_reg, 'area': area_reg, 'property_type': '公寓', 'decoration': {'$in': ['精装修', '豪华装修']}}
+
+    part_residential_query_args = {'date': date_reg, 'area': area_reg, 'property_type': '普通住宅', 'decoration': {'$in': ['精装修', '豪华装修']}}
+    part_department_query_args = {'date': date_reg, 'area': area_reg, 'property_type': '公寓', 'decoration': {'$in': ['精装修', '豪华装修']}}
+
+    city_residential_query_args_sold = {'date': date_reg, 'property_type': '普通住宅'}
+    city_department_query_args_sold = {'date': date_reg, 'property_type': '公寓'}
+
+    part_residential_query_args_sold = {'date': date_reg, 'property_type': '普通住宅'}
+    part_department_query_args_sold = {'date': date_reg, 'property_type': '公寓'}
+
+    task_args = [
+        city_residential_query_args,
+        city_department_query_args,
+        part_residential_query_args,
+        part_department_query_args,
+        city_residential_query_args_sold,
+        city_department_query_args_sold,
+        part_residential_query_args_sold,
+        part_department_query_args_sold
+                 ]
+
+    city_residential_average_rental = 0
+    city_department_average_rental = 0
+    part_residential_average_rental = 0
+    part_department_average_rental = 0
+    city_residential_average_sold = 0
+    city_department_average_sold = 0
+    part_residential_average_sold = 0
+    part_department_average_sold = 0
+
+    db = __open_db()
+    for i in range(len(task_args)):
+        col_name = ''
+        if i in [0, 1]:
+            col_name = 'rental_house'
+        elif i in [2, 3]:
+            col_name = 'sjk_rental_house'
+        elif i in [4, 5]:
+            col_name = 'nb_resold_house'
+        elif i in [6, 7]:
+            col_name = 'sjk_resold_house'
+
+        data_list = __query_data(col_name, task_args[i], db)
+        if data_list.count() == 0:
+            continue
+        average = __calculate_average_price(data_list)
+        if i == 0:
+            city_residential_average_rental = average
+        elif i == 1:
+            city_department_average_rental = average
+        elif i == 2:
+            part_residential_average_rental = average
+        elif i == 3:
+            part_department_average_rental = average
+        elif i == 4:
+            city_residential_average_sold = average
+        elif i == 5:
+            city_department_average_sold = average
+        elif i == 6:
+            part_residential_average_sold = average
+        elif i == 7:
+            part_department_average_sold = average
+
+    # 住宅出租均价
+    rental_residential_filter_args = {
+        'day': day_str,
+        'property_type': '1'
+    }
+    rental_residential_update_args = {
+        'city_price': city_residential_average_rental if city_residential_average_rental > 0 else 2290,
+        'part_price': part_residential_average_rental if part_residential_average_rental > 0 else 3100,
+        'month': month_str,
+        'update_date': time.strftime('%Y-%m-%d %H:%M:%S'),
+        'house_id': '109'
+    }
+
+    __update_data(db, 'rental_statistics', rental_residential_filter_args, rental_residential_update_args)
+
+    # 公寓出租均价
+    rental_department_filter_args = {
+        'day': day_str,
+        'property_type': '2'
+    }
+    rental_department_update_args = {
+        'city_price': city_department_average_rental if city_department_average_rental > 0 else 2330,
+        'part_price': part_department_average_rental if part_department_average_rental > 0 else 2850,
+        'month': month_str,
+        'update_date': time.strftime('%Y-%m-%d %H:%M:%S'),
+        'house_id': '109'
+    }
+
+    __update_data(db, 'rental_statistics', rental_department_filter_args, rental_department_update_args)
+
+    # 住宅出售均价
+    sold_residential_filter_args = {
+        'day': day_str,
+        'property_type': '1'
+    }
+    sold_residential_update_args = {
+        'city_price': city_residential_average_sold if city_residential_average_sold > 0 else 19920,
+        'part_price': part_residential_average_sold if part_residential_average_sold > 0 else 24450,
+        'month': month_str,
+        'update_date': time.strftime('%Y-%m-%d %H:%M:%S'),
+        'house_id': '109'
+    }
+
+    __update_data(db, 'sold_statistics', sold_residential_filter_args, sold_residential_update_args)
+
+    # 公寓出售均价
+    sold_department_filter_args = {
+        'day': day_str,
+        'property_type': '2'
+    }
+    sold_department_update_args = {
+        'city_price': city_department_average_sold if city_department_average_sold > 0 else 15650,
+        'part_price': part_department_average_sold if part_department_average_sold > 0 else 21560,
+        'month': month_str,
+        'update_date': time.strftime('%Y-%m-%d %H:%M:%S'),
+        'house_id': '109'
+    }
+
+    __update_data(db, 'sold_statistics', sold_department_filter_args, sold_department_update_args)
+
+    # 关闭数据库连接
+    db.client.close()
+
+    # 每日统计数据 通知谷晓晨
+    statistic_data = '宁波公寓租金: ' + str(city_department_average_rental) + '\n'
+    statistic_data += ('宁波住宅租金: ' + str(city_residential_average_rental) + '\n')
+    statistic_data += ('三江口公寓租金: ' + str(part_department_average_rental) + '\n')
+    statistic_data += ('三江口住宅租金: ' + str(part_residential_average_rental) + '\n')
+    statistic_data += ('宁波公寓售价: ' + str(city_department_average_sold) + '\n')
+    statistic_data += ('宁波住宅售价: ' + str(city_residential_average_sold) + '\n')
+    statistic_data += ('三江口公寓售价: ' + str(part_department_average_sold) + '\n')
+    statistic_data += ('三江口住宅售价: ' + str(part_residential_average_sold) + '\n')
+    email_util.send_email('每日统计数据', statistic_data, ['guxc@elab-plus.com', 'liuhx@elab-plus.com', 'zhub1@elab-plus.com'])
+
+
+# 计算平均值
+def __calculate_average_price(data_list):
+    price_list = []
+    for data in data_list:
+        try:
+            date_str = re.search(r'[0-9]{4,}', data['price']).group()
+            if date_str:
+                  price_list.append(float(date_str))
+
+        except Exception as err:
+            pass
+    sum = 0
+    for price in price_list:
+        sum += price
+    if len(price_list) == 0:
+        return 0
+    average = int(round(sum / len(price_list)))
+
+    return average
+
+
+# db 操作
+def __query_data(collection, query_args, db):
+    coll = db[collection]
+    data_list = coll.find(query_args).sort('data', -1)
+    return data_list
+
+
+# 更新数据
+def __update_data(db, collection, filter, update):
+    coll = db[collection]
+    coll.update_one(filter, {'$set': update}, upsert=True)
+
+
+# 打开数据库
+def __open_db():
+    # client = pymongo.MongoClient('101.132.106.154', authSource='logdb', authMechanism='SCRAM-SHA-1')
+    client = pymongo.MongoClient(MONGO_HOST, authSource='logdb')
+    db = client['logdb']
+    return db
+
+# list = [
+#     '2018-05-01',
+#     '2018-05-02',
+#     '2018-05-03',
+#     '2018-05-04',
+#     '2018-05-05',
+#     '2018-05-06',
+#     '2018-05-07',
+#     '2018-05-08',
+#     '2018-05-09',
+#     '2018-05-10',
+#     '2018-05-11',
+#     '2018-05-12',
+#     '2018-05-13',
+#     '2018-05-14',
+#     '2018-05-15',
+#     '2018-05-16',
+#     '2018-05-17',
+#     '2018-05-18',
+#     '2018-05-19',
+#     '2018-05-20',
+#     '2018-05-21',
+#     '2018-05-22'
+# ]
+#
+# for i in list:
+# daily_statistic()