elab_mvp: 提交报表推送的架构代码

Signed-off-by: binren <zhangbr@elab-plus.com>

apscheduler_elab.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+from panda_util import PandaUtil
+import time
+from email_util import EmailUtil
+
+
+class Config(object):
+    """
+        任务id对应的key
+        day_push：1
+        week_push_one：2
+        week_push_two： 3
+    """
+    JOBS = [
+        {
+            'id': 'day_push',
+            'func': 'apscheduler_elab:Funcs.day_push',
+            'args': '',
+            'trigger': 'cron',
+            'day_of_week': '*',
+            'hour': 9
+        },
+        {
+            'id': 'week_push_one',
+            'func': 'apscheduler_elab:Funcs.week_push_one',
+            'args': '',
+            'trigger': 'cron',
+            'day_of_week': '1',
+            'hour': 8,
+            'minute': 50
+        },
+        {
+            'id': 'week_push_two',
+            'func': 'apscheduler_elab:Funcs.week_push_two',
+            'args': '',
+            'trigger': 'cron',
+            'day_of_week': '1',
+            'hour': 10,
+            'minute': 50
+        }
+    ]
+
+    # 线程池配置
+    SCHEDULER_EXECUTORS = {
+        'default': {'type': 'threadpool', 'max_workers': 20}
+    }
+
+    SCHEDULER_JOB_DEFAULTS = {
+        'coalesce': False,
+        'max_instances': 3
+    }
+
+    # 调度器开关
+    SCHEDULER_API_ENABLED = True
+    pass
+
+
+class Funcs(object):
+    @staticmethod
+    def day_push():
+        print(time.time())
+
+    @staticmethod
+    def week_push_one():
+        print(time.time())
+
+    @staticmethod
+    def week_push_two():
+        print(time.time())
+
+    @staticmethod
+    def minute_push_elab():
+        pdu = PandaUtil('linshi')
+        sql = 'select house_id, COUNT(house_id) as number from t_house_image group by house_id limit 5'
+        file_name = 'pandas_chart_columns2{}.xlsx'.format(time.time())
+        df_data = pdu.query_data(sql)
+        print(df_data.size)
+        pdu.panda_chart([df_data], 1, 'title x', 'title y', file_name)
+        send_email = EmailUtil()
+        send_email.send_mail(mail_excel=file_name, content='elab数据报表推送服务')
+
+
+if __name__ == '__main__':
+    pass

email_util.py

@@ -0,0 +1,68 @@
+# coding=utf-8
+import smtplib
+from email.mime.text import MIMEText
+from email.header import Header
+from smtplib import SMTP_SSL
+from email.mime.image import MIMEImage
+from email.mime.multipart import MIMEMultipart
+from email.mime.application import MIMEApplication
+from email.mime.base import MIMEBase
+from email.encoders import encode_base64
+import traceback
+import os
+
+
+class EmailUtil(object):
+    host_server = 'smtp.exmail.qq.com'
+    sender_email = 'zhangbr@elab-plus.com'
+    pwd = '306492mnA'
+    send_name = 'elab'
+    receiver = ['1285211525@qq.com', '15773153135@163.com']
+
+    def __init__(self):
+        pass
+
+    def send_mail(self,
+                  mail_title='elab-test',
+                  content=None,
+                  mail_excel=None
+                  ):
+        try:
+            smtp = SMTP_SSL(self.host_server)
+            smtp.set_debuglevel(1)
+            smtp.ehlo(self.host_server)
+            smtp.login(self.sender_email, self.pwd)
+            msg = MIMEMultipart('related')
+            msg['Subject'] = Header(mail_title, 'utf-8')
+            msg['From'] = self.send_name
+            msgAlternative = MIMEMultipart('alternative')
+            msg.attach(msgAlternative)
+            if content:
+                textApart = MIMEText(content)
+                msg.attach(textApart)
+            if mail_excel:
+                part = MIMEBase('application', "vnd.ms-excel")
+                with open(mail_excel, 'rb') as fp:
+                    part.set_payload(fp.read())
+                    encode_base64(part)
+                    part.add_header('Content-Disposition', f'attachment; filename="{os.path.split(mail_excel)[1]}"')
+                    msg.attach(part)
+            for mail in self.receiver:
+                msg['To'] = mail
+                try:
+                    print(mail)
+                    smtp.sendmail(self.sender_email, mail, msg.as_string())
+                except Exception as e:
+                    smtp.sendmail(self.sender_email, mail, msg.as_string())
+                    print(str(e))
+            smtp.quit()
+            print('Success!')
+        except:
+            print('Error!')
+            traceback.print_exc()
+
+
+if __name__ == '__main__':
+    send_email = EmailUtil()
+    send_email.send_mail('elab_test', mail_excel=r'D:\elab\elab_mvp\resources\tongce1.xlsx')
+    pass

flask_app.py

@@ -3,9 +3,12 @@ from mvp import Mvp
 import json
 from test_info import TestInfo
 from tongce import TongCe
+from apscheduler_elab import Config
+from flask_apscheduler import APScheduler
 
 
 app = Flask(__name__)
+app.config.from_object(Config())
 
 
 @app.route('/score', methods=['GET', 'POST'])
@@ -210,6 +213,9 @@ def update_other_city():
 
 
 if __name__ == '__main__':
+    scheduler = APScheduler()
+    scheduler.init_app(app)
+    scheduler.start()
     app.run(
         host='0.0.0.0',
         port=5001

panda_util.py

@@ -0,0 +1,71 @@
+from mysql_db import MysqlDB
+import pandas as pd
+from email_util import EmailUtil
+
+
+class PandaUtil(object):
+    def __init__(self, db_name):
+        self.con = MysqlDB(db_name, db_type=1).con
+        pass
+
+    def query_data(self, sql):
+        df = pd.read_sql_query(sql, self.con)
+        return df
+
+    def panda_chart(self, df_list, cols, title_x, title_y, file_name):
+        """
+        data of narray
+        index of data_frame:  [0,1,2,3]
+        cols numbers of static columns
+        """
+
+        writer = pd.ExcelWriter(file_name, engine='xlsxwriter')
+        for i, df in enumerate(df_list):
+            # df = pd.DataFrame(data, index=None, columns=["姓名", "饱和度", "人力"])
+            sheet_name = f'Sheet{i}'
+            df.to_excel(writer, sheet_name=sheet_name, index=False)
+            workbook = writer.book
+            worksheet = writer.sheets[sheet_name]
+            chart = workbook.add_chart({'type': 'column'})
+            # set colors for the chart each type .
+            colors = ['#E41A1C', '#377EB8', '#4DAF4A', '#984EA3', '#FF7F00', '#7CFC00', '	#76EEC6', '#7EC0EE', '#00F5FF']
+            # Configure the series of the chart from the dataframe data.
+            for col_num in range(1, cols + 1):
+                chart.add_series({
+                    'name': [f'{sheet_name}', 0, col_num],
+                    'categories': [f'{sheet_name}', 1, 0, 4, 0],  # axis_x start row ,start col,end row ,end col
+                    'values': [f'{sheet_name}', 1, col_num, 4, col_num],  # axis_y value of
+                    'fill': {'color': colors[col_num - 1]},  # each type color choose
+                    'overlap': -10,
+                })
+
+            # Configure the chart axes.
+            chart.set_x_axis({'name': f'{title_x}'})
+            chart.set_y_axis({'name': f'{title_y}', 'major_gridlines': {'visible': False}})
+            chart.set_size({'width': 900, 'height': 400})
+            # Insert the chart into the worksheet.
+            worksheet.insert_chart('H2', chart)
+        writer.save()
+        writer.save()
+
+
+if __name__ == '__main__':
+    # pdu = PandaUtil('linshi')
+    # sql = 'select house_id, COUNT(house_id) as number from t_house_image group by house_id limit 5'
+    # df_data = pdu.query_data(sql)
+    # print(df_data.size)
+    # pdu.panda_chart([df_data], 1, 'title x', 'title y', 'pandas_chart_columns2.xlsx')
+    # send_email = EmailUtil()
+    # send_email.send_mail(mail_excel='pandas_chart_columns2.xlsx')
+    import pandas as pd
+    import numpy as np
+
+    df = pd.DataFrame({'ID': [1, 2, 3, None, 5, 6, 7, 8, 9, 10],
+                       'Name': ['Tim', 'Victor', 'Nick', None, 45, 48, '哈哈', '嗯呢', 'ess', 'dss'],
+                        'address': ['美国', '试试', '单独', None, '刚刚', '信息', '报表', '公司', '是否', '是否'],
+                       'address': ['美国', '试试', '单独', None, '刚刚', '信息', '报表', '公司', '是否', '是否'],
+                       'address': ['美国', '试试', '单独', None, '刚刚', '信息', '报表', '公司', '是否', '是否']
+                       }
+                      )
+    df.set_index("ID")
+    df.to_excel('output.xlsx')

report_push.py

@@ -0,0 +1,141 @@
+
+
+class ReportPush(object):
+    """
+        报表推送功能实现类
+    """
+    pass
+
+    # 数据一，周数据概览
+    sql_1 = """
+        待定
+    """
+
+    #  1.默认值/001_大麦/项目排行榜/小程序排行榜TOP_N
+    sql_2_1 = """
+        SELECT
+            a.*, 
+            b.house_name,
+            c.interested_num,
+            d.wx_num,
+            e.new_cust_num
+        FROM
+            (
+                SELECT
+                    house_id,
+                    count(
+                        DISTINCT ifnull(user_id, idfa)
+                    ) uv,
+                    sum(session_times) session_times,
+                    sum(sum_session_time) sum_session_time,
+                    sum(pv) pv,
+                    sum(page_num) page_num
+                FROM
+                    a_idfa_behavior_sum
+                WHERE
+                    report_d >= '2020-03-01'
+                AND report_d < '2020-03-05'
+                GROUP BY
+                    house_id
+            ) a
+        JOIN d_house b ON a.house_id = b.house_id
+        LEFT JOIN (
+            SELECT
+                house_id,
+                count(*) interested_num
+            FROM
+                f_interested_custlist
+            WHERE
+                report_d >= '2020-03-01'
+            AND report_d <'2020-03-05'
+            GROUP BY
+                house_id
+        ) c ON a.house_id = c.house_id
+        LEFT JOIN (
+            SELECT
+                house_id,
+                count(*) wx_num
+            FROM
+                f_customer_dynamic
+            WHERE
+                dynamic = 1
+            AND report_d >= '2020-03-01'
+            AND report_d <= '2020-03-05'
+            GROUP BY
+                house_id
+        ) d ON a.house_id = d.house_id
+        LEFT JOIN (
+            SELECT
+                house_id,
+                count(*) new_cust_num
+            FROM
+                d_user
+            WHERE
+                created >= '2020-03-01'
+            AND created < '2020-03-05' # 时间需要加一天！！！！
+            GROUP BY
+                house_id
+        ) e ON a.house_id = e.house_id
+    """
+
+    # 2.默认值/006_大麦（集团）/集团项目排行榜v1.3/集团排行榜
+    sql_2_2 = """
+    待定
+    """
+
+    # 默认值/001_大麦/场景_用户来源渠道/用户来源渠道—明细
+    sql_3_1 = """
+            SELECT
+                *
+            FROM
+                d_user_attr a
+            LEFT JOIN d_scene b ON a.scene = b. CODE
+            WHERE
+                a.source IN (1, 2, 3, 4, 10)
+            AND a.report_d >= '2020-03-01'
+            AND a.report_d <= '2020-03-04'
+    """
+
+    # 默认值/006_大麦（集团）/场景(集团)_用户来源渠道_v1.1/用户来源渠道—明细
+    sql_3_2 = """
+            SELECT
+                a.scene,
+                a.brand_id,
+                b.*, a.share_brand_customer_id,
+                '2' adviser_agent,
+                a.house_id house_id,
+                c.house_name house_name,
+                c.brand_name
+            FROM
+                (
+                    SELECT
+                        scene,
+                        brand_id,
+                        share_brand_customer_id,
+                        house_id
+                    FROM
+                        d_brand_app_customer
+                    WHERE
+                        created >='2020-03-01'
+                    AND created < DATE_ADD(
+                        '2020-03-04', INTERVAL 1 DAY
+                    )
+                    UNION ALL
+                        SELECT
+                            scene,
+                            brand_id,
+                            share_brand_customer_id,
+                            brand_id house_id
+                        FROM
+                            d_brand_app_customer
+                        WHERE
+                            created >= '2020-03-01'
+                        AND created < DATE_ADD(
+                            '2020-03-04', INTERVAL 1 DAY
+                        )
+                ) a
+            LEFT JOIN d_scene b ON a.scene = b. CODE
+            JOIN d_house c ON a.house_id = c.house_id
+            AND a.brand_id = c.brand_id
+    """
+