mvp.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800
  1. from mysql_db import MysqlDB
  2. from excel_util import ExcelUtil
  3. import time
  4. from entity import PeopleInfo
  5. class Mvp:
  6. """
  7. ce mvp 答题数据统计
  8. 城市特例 北京市,上海市, 重庆市,天津市
  9. """
  10. age_dict = {
  11. '00-04年生': '00后',
  12. '05-09年生': '05后',
  13. '50-59年生': '50后',
  14. '60-69年生': '60后',
  15. '70-74年生': '70后',
  16. '75-79年生': '75后',
  17. '80-84年生': '80后',
  18. '85-89年生': '85后',
  19. '90-94年生': '90后',
  20. '95-99年生': '95后'
  21. }
  22. age_list = ['85后', '95后']
  23. city_list = ['上海市', '上海周边']
  24. tag_table = {
  25. '用户画像-审美偏好': ['mvp_crowd_info_aesthetic_preference', 'aesthetic_preference'],
  26. '用户画像-行为兴趣': ['mvp_crowd_info_behavior', 'behavioral_interest'],
  27. '用户画像-观念': ['mvp_crowd_info_consumer_concept', ''],
  28. '用户画像-消费特征': ['mvp_crowd_info_consumer_structure', ''],
  29. '空间需求图谱-功能关联': ['mvp_crowd_info_functional_module', ''],
  30. '性别比例': ['mvp_crowd_info_gender_rate', ''],
  31. '用户画像-生活方式': ['mvp_crowd_info_life_style', ''],
  32. '人群占比': ['mvp_crowd_info_rate', ''],
  33. '用户画像-社交模式': ['mvp_crowd_info_social_mode', ''],
  34. '用户画像-行业': ['mvp_crowd_info_trade', ''],
  35. '用户画像-出行方式': ['mvp_crowd_info_trip_mode', ''],
  36. '空间需求图谱-基础模块分值': ['mvp_innovate_space_base_module', ''],
  37. '空间需求图谱-色相': ['mvp_innovate_space_color_prefer', 'color'],
  38. '空间需求图谱-精装关注点': ['mvp_innovate_space_hardcover_focus', 'hardcover_focus'],
  39. '空间需求图谱-色调': ['mvp_innovate_space_hue_prefer', 'hue'],
  40. '空间需求图谱-单品偏好': ['mvp_innovate_space_item_preference', 'item_preference'],
  41. '空间需求图谱-材质': ['mvp_innovate_space_material_prefer', 'material'],
  42. '空间需求图谱-空间特性偏好': ['mvp_innovate_space_space_prefer', 'space_preference'],
  43. '空间需求图谱-空间拓普图': ['mvp_innovate_space_space_top', ''],
  44. '模块分数': ['mvp_crowd_info_module', 'module_name']
  45. }
  46. crowd_info_1 = {
  47. '1973': 'A',
  48. '1974': 'B',
  49. '1975': 'C',
  50. '1976': 'D',
  51. '1977': 'E',
  52. '1978': 'F',
  53. '1979': 'G',
  54. '1813': 'A',
  55. '1814': 'B',
  56. '1815': 'C',
  57. '1816': 'D',
  58. '1817': 'E',
  59. '1818': 'F',
  60. '1819': 'G'
  61. }
  62. base_insert_sql = 'insert into {}(crowd_info_id, {}, standard_value, status) values(%s, %s, %s, ' \
  63. '1) '
  64. def get_table_name(self, name):
  65. """
  66. 获取表名
  67. :param name:
  68. :return:
  69. """
  70. params = self.tag_table.get(name)
  71. if params:
  72. return self.tag_table.get(name)[0]
  73. def get_insert_sql(self, tag_type_name):
  74. """
  75. 根据标签分类名称获取相应表的插入sql
  76. :param tag_type_name:
  77. :return:
  78. """
  79. params = self.tag_table.get(tag_type_name)
  80. if params:
  81. return self.base_insert_sql.format(params[0], [1])
  82. crowd = ['A', 'B', 'C', 'D', 'E', 'F']
  83. # 获取答题记录中城市列表
  84. sql_1 = 'select city from f_t_daren_score_2 group by city'
  85. # 获取父选项和父题id
  86. sql_2 = 'select a.id, a.content, b.id, b.name from bq_option a left join bq_question b on a.question_id = b.id ' \
  87. 'where a.serial_number = %s and b.serial_number = %s and a.status = b.status = 1 '
  88. # 获取答题人的年龄段集合
  89. sql_4 = 'select nld from f_t_daren_score_2 group by nld'
  90. # 根据城市,年龄段,人群分类统计答题记录数
  91. sql_5 = 'select testcase_id, COUNT(uuid) from f_t_daren_score_2 where uuid in %s group by testcase_id '
  92. # 根据父选项获取子选项id列表
  93. sql_6 = '''
  94. SELECT
  95. c.id,
  96. c.sub_question_id,
  97. c.content
  98. FROM
  99. bq_sub_option c
  100. WHERE
  101. c.father_id IN (
  102. SELECT
  103. a.id
  104. FROM
  105. bq_option a
  106. LEFT JOIN bq_question b ON a.question_id = b.id
  107. WHERE
  108. a.serial_number = % s
  109. AND b.serial_number = % s
  110. AND a. STATUS = 1
  111. AND b. STATUS = 1
  112. )
  113. AND c. STATUS = 1
  114. '''
  115. # 根据子题id获取包含子题id的测试
  116. sql_7 = 'select id from bq_testcase where status = 1 and FIND_IN_SET(%s, question_ids)'
  117. # 根据子选项id统计答题数
  118. sql_8 = '''
  119. SELECT
  120. count(1)
  121. FROM
  122. f_t_daren_score_2 a
  123. LEFT JOIN d_shangju_tiku_02 b ON a.sub_question_id = b.sub_question_id
  124. AND (
  125. a.score = b.score
  126. OR a.score = b.sub_option_id
  127. )
  128. AND a.testcase_id = b.testcase_id
  129. WHERE
  130. b.sub_option_id IN % s
  131. AND a.uuid IN % s
  132. '''
  133. # 获取一个uuid下答题的子选项id列表
  134. sql_10 = 'select DISTINCT uuid, GROUP_CONCAT(DISTINCT b.sub_option_id) from f_t_daren_score_2 a left join ' \
  135. 'd_shangju_tiku_02 b on a.sub_question_id = b.sub_question_id and (a.score = b.score or a.score = ' \
  136. 'b.sub_option_id) where a.status = ' \
  137. 'b.status = 1 group by uuid '
  138. # 向表mvp_crowd_info插入数据
  139. sql_11 = 'insert into mvp_crowd_info(age_area, city_name, crowd_type, status) values(%s, %s, %s, 1)'
  140. # 向表mvp_crowd_info_behavior中插入数据
  141. sql_12 = 'insert into mvp_crowd_info_behavior(crowd_info_id, behavioral_interest, standard_value, status) values(' \
  142. '%s, %s, ' \
  143. '%s, 1) '
  144. # 向表mvp_crowd_info_module中插入数据
  145. sql_13 = 'insert into mvp_crowd_info_module(crowd_info_id, module_name, standard_value, status) values (%s, %s, ' \
  146. '%s, 1) '
  147. sql_14 = 'select a.id, a.age_area, a.city_name, a.crowd_type from mvp_crowd_info a where a.status = 1'
  148. # 获取答题城市信息from city
  149. sql_15 = '''
  150. SELECT
  151. a.uuid,
  152. IFNULL(GROUP_CONCAT(DISTINCT a.city, a.province), 00) AS city,
  153. IFNULL(GROUP_CONCAT(DISTINCT a.nld), 00) AS nld,
  154. IFNULL(GROUP_CONCAT(DISTINCT a.sex), 00) AS sex,
  155. IFNULL(GROUP_CONCAT(DISTINCT b.sub_option_id), 00) as sub_option_ids,
  156. IFNULL(GROUP_CONCAT(DISTINCT a.testcase_id), 00) as testcase_ids
  157. FROM
  158. f_t_daren_score_2 a
  159. LEFT JOIN d_shangju_tiku_02 b ON a.testcase_id = b.testcase_id
  160. WHERE
  161. a.testcase_id = b.testcase_id
  162. AND a.sub_question_id = b.sub_question_id
  163. AND (
  164. a.score = b.score
  165. OR a.score = b.sub_option_id
  166. )
  167. GROUP BY
  168. a.uuid
  169. '''
  170. # 根据用户uuid获取城市信息
  171. sql_16 = '''
  172. SELECT
  173. a.uuid,
  174. b.sub_option_content
  175. FROM
  176. f_t_daren_score_2 a
  177. LEFT JOIN d_shangju_tiku_02 b ON a.testcase_id = b.testcase_id
  178. WHERE
  179. a.sub_question_id = b.sub_question_id
  180. AND (
  181. a.score = b.score
  182. OR a.score = b.sub_option_id
  183. )
  184. AND a.uuid = %s
  185. AND b.father_id = 249
  186. AND a. STATUS = b. STATUS = 1
  187. '''
  188. # 答题人人群分类信息
  189. sql_17 = '''
  190. SELECT
  191. a.uuid,
  192. b.sub_option_id
  193. FROM
  194. f_t_daren_score_2 a
  195. LEFT JOIN d_shangju_tiku_02 b ON a.testcase_id = b.testcase_id
  196. WHERE
  197. a.sub_question_id = b.sub_question_id
  198. AND (
  199. a.score = b.score
  200. OR a.score = b.sub_option_id
  201. )
  202. AND a.uuid = %s
  203. AND b.father_id = 236
  204. AND a.STATUS = b.STATUS = 1
  205. '''
  206. """
  207. 数据debug SQL
  208. 1:
  209. SELECT
  210. c.id,
  211. c.sub_question_id,
  212. c.content
  213. FROM
  214. bq_sub_option c
  215. WHERE
  216. c.father_id IN (
  217. SELECT
  218. a.id
  219. FROM
  220. bq_option a
  221. LEFT JOIN bq_question b ON a.question_id = b.id
  222. WHERE
  223. a.serial_number ='FA001'
  224. AND b.serial_number = 'F00245'
  225. AND a. STATUS = 1
  226. AND b. STATUS = 1
  227. )
  228. AND c.STATUS = 1
  229. 2:
  230. select id from bq_testcase where status = 1 and FIND_IN_SET(%s, question_ids)
  231. 3:
  232. SELECT
  233. count(1)
  234. FROM
  235. f_t_daren_score_2 a
  236. LEFT JOIN d_shangju_tiku_02 b ON a.sub_question_id = b.sub_question_id
  237. AND (
  238. a.score = b.score
  239. OR a.score = b.sub_option_id
  240. )
  241. AND a.testcase_id = b.testcase_id
  242. WHERE
  243. b.sub_option_id IN (1964,1965,1966,1967,1968,1969,1970,1971,1972)
  244. """
  245. def __init__(self, path=None):
  246. self.shangju_db = MysqlDB('shangju')
  247. self.marketing_db = MysqlDB('bi_report')
  248. # self.shangju_db.truncate('mvp_standard_score')
  249. self.tag_data = ExcelUtil(file_name=path).init_mvp_data()
  250. self.crowd_info = ExcelUtil(file_name=path, sheet_name='选项-人群分类对应表').init_crowd_info()
  251. self.citys = self.init_city()
  252. self.age = self.init_age()
  253. self.people_sub_option_ids = self.marketing_db.select(self.sql_10)
  254. self.crowd_contain_sub_option_ids = self.get_crowd_contain_sub_option_ids()
  255. self.module_scores = ExcelUtil(file_name='set-behavior-tag.xlsx', sheet_name='算法关系表').init_module_info()
  256. # self.scores_tag = ExcelUtil(file_name='行为与模块分值汇总.xlsx', sheet_name='行为').init_scores()
  257. # self.score_module = ExcelUtil(file_name='行为与模块分值汇总.xlsx', sheet_name='模块').init_scores()
  258. self.scores_tag = None
  259. self.score_module = None
  260. def close(self):
  261. self.shangju_db.close()
  262. self.marketing_db.close()
  263. def init_city(self):
  264. """
  265. 获取答题数据中的城市。
  266. :return:
  267. """
  268. citys = ['上海市', '上海周边']
  269. # citys_info = self.marketing_db.select(self.sql_1)
  270. # citys.extend([x[0] for x in citys_info if x[0] is not None])
  271. return citys
  272. def query_behavioral_info(self, city=None, age=None, crowd=None):
  273. """
  274. 查询行为兴趣信息
  275. :return:
  276. """
  277. # datas = []
  278. # for key in self.tag_data.keys():
  279. # values = self.tag_data[key]
  280. # for value in values:
  281. # question = value[0].split('-')[0]
  282. # option = value[0].split('-')[1]
  283. # corr = value[1]
  284. # data = self.shangju_db.select(self.sql_2, [option, question])
  285. # if len(data) > 0:
  286. # print([question, option, data[0][3], data[0][1], key, corr])
  287. # datas.append([question, option, data[0][3], data[0][1], key, corr])
  288. # self.shangju_db.truncate('mvp_question_classification')
  289. # self.shangju_db.add_some(self.sql_3, datas)
  290. scores_behavioral = self.city_age_crowd(city, age, crowd)
  291. # scores_module = self.module_score(crowd, city, age, scores_behavioral['score'])
  292. # result = {'行为兴趣分值': scores_behavioral['score'], '模块分值': scores_module}
  293. print('update finished!!!')
  294. return scores_behavioral
  295. def people_info(self):
  296. """
  297. 答题人个人信息获取
  298. :return:
  299. """
  300. people_info_city = self.marketing_db.select(self.sql_15)
  301. people_infos = []
  302. for people in people_info_city:
  303. uuid = people[0]
  304. city = people[1]
  305. nld = people[2]
  306. sex = people[3]
  307. sub_option_ids_1 = people[4]
  308. testcaseid = people[5]
  309. if str(city).find('市') != -1:
  310. city = str(city).split('市')[0] + '市'
  311. if str(nld).find(',') != -1:
  312. nld_1 = list(str(nld).split(','))
  313. if len(nld_1) > 0:
  314. nld = nld_1[0]
  315. else:
  316. pass
  317. crowd = []
  318. if testcaseid:
  319. testcastids = list(map(int, str(testcaseid).split(',')))
  320. if len(testcastids) > 0:
  321. gt_75 = [x for x in testcastids if x > 74]
  322. if len(gt_75) > 0:
  323. # 从答题结果中获取城市信息
  324. citys = self.marketing_db.select(self.sql_16, [uuid])
  325. if len(citys) > 0:
  326. city = '上海市' if citys[0][1] == '一线' else '上海周边'
  327. # 根据用户子选项id集合,获取用户的人群分类
  328. if len(gt_75) > 0:
  329. # 特定的测试人群分类从答题结果中获取
  330. sub_option_ids = self.marketing_db.select(self.sql_17, [uuid])
  331. for option in sub_option_ids:
  332. crowd_type = self.crowd_info_1.get(option[1])
  333. if crowd_type:
  334. crowd.append(crowd_type)
  335. else:
  336. crowd.append('A')
  337. else:
  338. if sub_option_ids_1 is not None:
  339. crowd.extend(self.get_people_uuid_by_sub_option_ids(sub_option_ids_1))
  340. if city is None:
  341. city = '上海市'
  342. people_info = PeopleInfo(uuid, city, nld, sex, crowd)
  343. people_infos.append(people_info)
  344. # people_infos.append([uuid, city, nld, sex, crowd])
  345. return people_infos
  346. def people_filter(self, city, nld, crowd):
  347. uuids = []
  348. for people in self.people_info_1:
  349. if people.city == city and people.age == nld and crowd in people.crowd:
  350. uuids.append(people.uuid)
  351. return uuids
  352. def get_people_uuid_by_sub_option_ids(self, sub_ids):
  353. types = []
  354. for key in self.crowd_contain_sub_option_ids.keys():
  355. type_sub_option_ids = self.crowd_contain_sub_option_ids[key]
  356. sub_option_ids = list(map(int, str(sub_ids).split(',')))
  357. # list(set(a).intersection(set(b)))
  358. if len(list(set(sub_option_ids).intersection(set(type_sub_option_ids)))) > 0 and key not in types:
  359. types.append(key)
  360. return types
  361. def update_data(self):
  362. """
  363. 定时更新分值
  364. :return:
  365. """
  366. self.linshi_db = MysqlDB('linshi', db_type=1)
  367. for city in self.city_list:
  368. for age in self.age_list:
  369. for crowd in self.crowd:
  370. result = self.city_age_crowd(city, age, crowd)
  371. self.insert_score_to_db(result)
  372. print('{}数据关系完成...'.format(time.time()))
  373. def insert_score_to_db(self, scores):
  374. """
  375. 行为、模块分数写入数据库
  376. :return:
  377. """
  378. ids = self.query_data()
  379. behavior_score = scores['behavior_score']
  380. module_score = scores['module_score']
  381. module_insert_sql = self.get_insert_sql('模块分数')
  382. if module_insert_sql:
  383. module_insert_data = []
  384. for module in module_score:
  385. city_2 = module[0]
  386. age_2 = module[1]
  387. crowd_2 = module[2]
  388. module_name_2 = module[3]
  389. module_score_2 = module[4]
  390. for id in ids:
  391. city_1 = id[2]
  392. age_1 = id[1]
  393. crowd_1 = id[3]
  394. id_1 = id[0]
  395. if city_2 == city_1 and self.age_dict[age_2] == age_1 and crowd_2 == crowd_1:
  396. module_insert_data.append([id_1, module_name_2, module_score_2])
  397. # 先清空之前的数据
  398. table_name = self.get_table_name('模块分数')
  399. if table_name and len(module_insert_data) > 0:
  400. self.linshi_db.truncate(table_name)
  401. self.linshi_db.add_some(module_insert_sql, module_insert_data)
  402. print('模块分数更新完成...')
  403. for b_score in behavior_score:
  404. for key in b_score.keys():
  405. insert_sql = self.get_insert_sql(key)
  406. if insert_sql:
  407. insert_data = []
  408. score = b_score[key]
  409. for data in score:
  410. city = data[0]
  411. age = data[1]
  412. tag_name = data[2]
  413. crowd = data[3]
  414. tag_score = data[4]
  415. for id in ids:
  416. city_1 = id[2]
  417. age_1 = id[1]
  418. crowd_1 = id[3]
  419. id_1 = id[0]
  420. if city == city_1 and self.age_dict[age] == age_1 and crowd == crowd_1:
  421. insert_data.append([id_1, tag_name, tag_score])
  422. if len(insert_data) > 0:
  423. table_name = self.get_table_name(key)
  424. if table_name:
  425. self.linshi_db.truncate(table_name)
  426. self.linshi_db.add_some(insert_sql, insert_data)
  427. else:
  428. print('未找到对应的表,数据无法插入...')
  429. print('行为分数更新完成...')
  430. def module_score(self, crowd, city, age, scores):
  431. """
  432. 模块分数计算
  433. 城市 年龄 人群分类 模块名称 分数
  434. :return:
  435. """
  436. # import json
  437. # print(json.dumps(scores, ensure_ascii=False))
  438. modules = self.module_scores[crowd]
  439. result = []
  440. for key in modules.keys():
  441. values = modules[key]
  442. module_name = key
  443. score = 0
  444. for value in values:
  445. behavioral_name = value[0]
  446. weight = float(value[2])
  447. standard_score = [x[4] for x in scores if x[2] == behavioral_name]
  448. if len(standard_score) > 0:
  449. score += standard_score[0] * weight
  450. result.append([city, age, crowd, module_name, score])
  451. return result
  452. # def insert_data(self, scores_behavioral, scores_module):
  453. def insert(self):
  454. """
  455. 计算数据写入数据库中,供接口查看
  456. :return:
  457. """
  458. infos = []
  459. for city in self.city_list:
  460. for age in self.age_list:
  461. for c_type in self.crowd:
  462. age_area = self.age_dict.get(age)
  463. if age_area:
  464. infos.append([age_area, city, c_type])
  465. self.shangju_db.add_some(self.sql_11, infos)
  466. def query_data(self):
  467. ids = self.linshi_db.select(self.sql_14)
  468. return ids
  469. def shanghai_85_module_score_insert(self):
  470. """
  471. 上海市,85后模块分数计算
  472. :return:
  473. """
  474. result = []
  475. for crowd in self.crowd:
  476. modules = self.module_scores[crowd]
  477. for key in modules.keys():
  478. values = modules[key]
  479. module_name = key
  480. score = 0
  481. for value in values:
  482. behavioral_name = value[0]
  483. weight = float(value[2])
  484. # standard_score = [x[4] for x in scores if x[2] == behavioral_name]
  485. standard_score = float(value[1])
  486. if standard_score is not None:
  487. score += standard_score * weight
  488. result.append(['上海市', '85后', crowd, module_name, score])
  489. return {'score': result, 'data': self.module_scores}
  490. def tag_module_score_insert(self):
  491. """
  492. 标签模块分数写入数据库
  493. :return:
  494. """
  495. ids = self.query_data()
  496. insert_data = []
  497. insert_data_1 = []
  498. for tag, module in zip(self.scores_tag, self.score_module):
  499. city = tag[0]
  500. age = tag[1]
  501. crowd = tag[2]
  502. tag_name = tag[3]
  503. tag_score = tag[4]
  504. city_2 = module[0]
  505. age_2 = module[1]
  506. crowd_2 = module[2]
  507. module_name_2 = module[3]
  508. module_score_2 = module[4]
  509. for id in ids:
  510. city_1 = id[2]
  511. age_1 = id[1]
  512. crowd_1 = id[3]
  513. id_1 = id[0]
  514. if city == city_1 and self.age_dict[age] == age_1 and crowd == crowd_1:
  515. insert_data.append([id_1, tag_name, tag_score])
  516. if city_2 == city_1 and self.age_dict[age_2] == age_1 and crowd_2 == crowd_1:
  517. insert_data_1.append([id_1, module_name_2, module_score_2])
  518. self.shangju_db.add_some(self.sql_12, insert_data)
  519. self.shangju_db.add_some(self.sql_13, insert_data_1)
  520. def init_age(self):
  521. """
  522. 获取答题数据中的年龄
  523. """
  524. return ['95后', '85后']
  525. # age_info = self.marketing_db.select(self.sql_4)
  526. # # print([x[0] for x in age_info])
  527. # return [x[0] for x in age_info if x[0] is not None]
  528. def city_age_crowd(self, city=None, age=None, crowd=None):
  529. data_start = []
  530. result = []
  531. module_scores = []
  532. self.people_info_1 = self.people_info()
  533. if city is not None and age is not None and crowd is not None:
  534. print('获取指定城市,年龄段,人群类型的数据...')
  535. # people_uuids = self.get_people_uuid_by_type(crowd)
  536. people_uuids = self.people_filter(city, age, crowd)
  537. behavior_data = None
  538. if len(people_uuids) > 0:
  539. print('{}-{}-{}'.format(city, age, crowd))
  540. datas = self.behavior_tag_init(city, age, people_uuids)
  541. data_start.append(datas)
  542. all_data, behavior_data_1 = self.calculation_standard_score(datas, city, age, crowd)
  543. result.append(all_data)
  544. behavior_data = behavior_data_1
  545. if behavior_data:
  546. module_scores.extend(self.module_score(crowd, city, age, behavior_data))
  547. # data_list = []
  548. # for e in data_start:
  549. # for key in e.keys():
  550. # values = e[key]
  551. # for sub_e in values:
  552. # ele = [key]
  553. # ele.extend(sub_e)
  554. # data_list.append(ele)
  555. # pass
  556. return {'behavior_score': result, 'module_score': module_scores}
  557. # return {'score': result, 'data': data_list}
  558. def scores(self):
  559. behavior_score = []
  560. module_scores = []
  561. for city in self.city_list:
  562. for age in self.age_list:
  563. for crowd in self.crowd:
  564. data = self.city_age_crowd(city, age, crowd)
  565. behavior_score.extend(data['behavior_score'])
  566. module_scores.extend(data['module_score'])
  567. return {'behavior_score': behavior_score, 'module_score': module_scores}
  568. def behavior_tag_init(self, city, age, people_uuids):
  569. result = {}
  570. self.group_type_count = self.marketing_db.select(self.sql_5, [people_uuids])
  571. # 表名
  572. for key in self.tag_data.keys():
  573. values = self.tag_data[key]
  574. result_sub = {}
  575. # 标签
  576. for key_tag_name in values.keys():
  577. questions = values[key_tag_name]
  578. elements = []
  579. for value in questions:
  580. question = value[0].split('-')[0]
  581. option = value[0].split('-')[1]
  582. corr = value[1]
  583. fz, fm = self.molecular_value(question, option, city, age, people_uuids)
  584. if fm == 0:
  585. c = 0
  586. else:
  587. c = fz / fm
  588. elements.append([question, option, corr, fz, fm, c])
  589. result_sub[key_tag_name] = elements
  590. result[key] = self.indicator_calculation_d_e(result_sub)
  591. return result
  592. def molecular_value(self, queston, option, city, age, people_uuids):
  593. # 获取当前父选项包含的子选项id和子题id列表
  594. result = self.shangju_db.select(self.sql_6, [option, queston])
  595. sub_option_ids = []
  596. group_types = []
  597. for rt in result:
  598. sub_option_id, sub_question_id, content = rt[0], rt[1], rt[2]
  599. grouptypes = self.shangju_db.select(self.sql_7, [sub_question_id])
  600. for g_t in grouptypes:
  601. if str(g_t[0]) not in group_types:
  602. group_types.append(str(g_t[0]))
  603. sub_option_ids.append(sub_option_id)
  604. # 计算子选项在答题记录中的点击数
  605. sub_options_count = 0
  606. if len(sub_option_ids) > 0:
  607. result_1 = self.marketing_db.select(self.sql_8, [sub_option_ids, people_uuids])
  608. sub_options_count = result_1[0][0]
  609. # 计算父选项包含的子选项对应的子题所在的测试gt包含的点击数。
  610. denominator_value = 0
  611. for info in self.group_type_count:
  612. if str(info[0]) in group_types:
  613. denominator_value += info[1]
  614. return sub_options_count, denominator_value
  615. def indicator_calculation_d_e(self, data):
  616. result = {}
  617. for key in data.keys():
  618. values = data[key]
  619. c_list = []
  620. for x in values:
  621. _x = x[5]
  622. if _x is not None and x != 0:
  623. c_list.append(_x)
  624. fm_list = [x[4] for x in values]
  625. sum_c = sum(fm_list)
  626. if len(c_list) == 0:
  627. min_c = 0
  628. else:
  629. min_c = min(c_list)
  630. elements = []
  631. for value in values:
  632. _value = []
  633. c = value[5]
  634. if sum_c == 0:
  635. d = 0
  636. else:
  637. d = c / sum_c
  638. e = c - min_c
  639. _value.extend(value)
  640. _value.append(d)
  641. _value.append(e)
  642. elements.append(_value)
  643. result[key] = elements
  644. return result
  645. def calculation_standard_score(self, datas, city, age, crowd_type):
  646. scores = {}
  647. for key_tag_type in datas.keys():
  648. print(key_tag_type)
  649. tag_type_data = datas[key_tag_type]
  650. scores_sub = []
  651. for key_tag in tag_type_data.keys():
  652. key_tag_data = tag_type_data[key_tag]
  653. print(key_tag)
  654. print(' 父题序号 父选项序号 相关系系数 分子值 分母值 百分比 人数权重 偏离值')
  655. values = [x[5] for x in key_tag_data]
  656. min_c = min(values)
  657. f = min_c
  658. for value in key_tag_data:
  659. print(' {}'.format(value))
  660. if value[2] is not None and value[7] is not None:
  661. f += float(value[2] * value[7])
  662. print(' 标准分:{}'.format(f))
  663. scores_sub.append([city, age, key_tag, crowd_type, f])
  664. scores[key_tag_type] = scores_sub
  665. # self.shangju_db.add_some(self.sql_9, scores)
  666. return scores, scores['用户画像-行为兴趣']
  667. def people_data(self):
  668. result = self.people_info()
  669. a = 0
  670. b = 0
  671. c = 0
  672. d = 0
  673. e = 0
  674. f = 0
  675. result_1 = []
  676. for rt in result:
  677. crowds = rt.crowd
  678. if rt.uuid in [
  679. 'ae9db26b-3606-497c-83c5-56341d487a91',
  680. '9fb33b6c-bd7a-4114-b225-3ee380943517',
  681. '84636488-1307-47fe-a238-4f9cf279a908',
  682. '4a5b6654-eb99-46ed-8dcf-777648d6baca',
  683. 'ba181da0-c91a-4430-84c6-9612a693f659',
  684. '32eae583-474c-4dca-8b36-d74314f45cee',
  685. 'b07f6ff2-ccd5-44ee-9b7c-b2e1f40d777f',
  686. '149a0e40-5639-4771-8a27-60821e14a1d5',
  687. '4795b731-3e75-4f08-90bc-8ee4a0c366c6',
  688. '4795b731-3e75-4f08-90bc-8ee4a0c366c6',
  689. '47cbd398-1c39-4dc0-8d97-98fe19457516']:
  690. result_1.append([rt.uuid, rt.city, rt.age, rt.crowd])
  691. if 'A' in crowds:
  692. a += 1
  693. if 'B' in crowds:
  694. b += 1
  695. if 'C' in crowds:
  696. c += 1
  697. if 'D' in crowds:
  698. d += 1
  699. if 'E' in crowds:
  700. e += 1
  701. if 'F' in crowds:
  702. f += 1
  703. return result_1
  704. # return {'A': a, 'B': b, 'C': b, 'D': d, 'E': e, 'F': f}
  705. def get_crowd_people(self):
  706. result = {}
  707. for type in self.crowd:
  708. uuids = self.get_people_uuid_by_type(type)
  709. result[type] = len(uuids)
  710. return result
  711. def get_people_uuid_by_type(self, type):
  712. uuids = []
  713. type_sub_option_ids = self.crowd_contain_sub_option_ids[type]
  714. for people in self.people_sub_option_ids:
  715. uuid = people[0]
  716. sub_option_ids = list(map(int, str(people[1]).split(',')))
  717. # list(set(a).intersection(set(b)))
  718. if len(list(set(sub_option_ids).intersection(set(type_sub_option_ids)))) > 0 and uuid not in uuids:
  719. uuids.append(uuid)
  720. return uuids
  721. def get_crowd_contain_sub_option_ids(self):
  722. """
  723. 获取ABCDEF人群包含的子选项id
  724. :return:
  725. """
  726. infos = {}
  727. for key in self.crowd_info.keys():
  728. values = self.crowd_info[key]
  729. sub_option_ids = []
  730. for value in values:
  731. if value is not None:
  732. vals = str(value).split('-')
  733. option, question = vals[1], vals[0]
  734. query_result = self.shangju_db.select(self.sql_6, [option, question])
  735. for qr in query_result:
  736. sub_option_id, sub_question_id, content = qr[0], qr[1], qr[2]
  737. sub_option_ids.append(int(sub_option_id))
  738. infos[key] = sub_option_ids
  739. return infos
  740. if __name__ == '__main__':
  741. mvp = Mvp()
  742. mvp.people_info()