From 1132ef28c09eb52714fa8737171e22928ad10678 Mon Sep 17 00:00:00 2001 From: Julin Date: Tue, 30 May 2023 10:20:45 +0800 Subject: [PATCH] append --- code/pep-stats-report/Dockerfile | 11 ++- code/pep-stats-report/app/db_helper.py | 50 +++++++--- code/pep-stats-report/app/image_helper.py | 110 ++++++++++++++++++++++ code/pep-stats-report/config.ini | 2 +- code/pep-stats-report/main.py | 78 +++++++++++++-- code/pep-stats-report/requirements.txt | 2 + 6 files changed, 229 insertions(+), 24 deletions(-) diff --git a/code/pep-stats-report/Dockerfile b/code/pep-stats-report/Dockerfile index f9b5f50..89ff1ff 100644 --- a/code/pep-stats-report/Dockerfile +++ b/code/pep-stats-report/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /app COPY . . RUN pip install --upgrade pip -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com && \ - pip install -r requirements.txt -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com && \ + pip install -r requirements_linux.txt -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com && \ cp SimHei.ttf /usr/local/lib/python3.8/site-packages/matplotlib/mpl-data/fonts/ttf && \ pyinstaller -F main.py --distpath=. @@ -13,6 +13,13 @@ FROM debian:11-slim WORKDIR /app -COPY --from=python_base /app/main /app/config.ini ./ +COPY --from=python_base /app/main /app/config.ini /app/SimHei.ttf ./ + +RUN cp -a /etc/apt/sources.list /etc/apt/sources.list.bak && \ + sed -i "s@http://deb.debian.org@http://mirrors.aliyun.com@g" /etc/apt/sources.list && \ + sed -i "s@http://security.debian.org@http://mirrors.aliyun.com@g" /etc/apt/sources.list + +RUN apt-get update && apt-get install libreoffice -y && \ + cp SimHei.ttf /usr/share/fonts/ CMD ["./main"] diff --git a/code/pep-stats-report/app/db_helper.py b/code/pep-stats-report/app/db_helper.py index 43aa29b..377c60a 100644 --- a/code/pep-stats-report/app/db_helper.py +++ b/code/pep-stats-report/app/db_helper.py @@ -1,5 +1,5 @@ def querystring_user_procinst_duration(start, end, except_senior, department): - filter_user = "and u.name not in ('刘文峰', '金亮', '姜珍', '余莎莎', '张阳根', '唐国华', '刘国勇', '刘会连', '肖琥')" if except_senior else "" + filter_user = "and u.name not in ('刘文峰', '金亮', '姜珍', '余莎莎', '张阳根', '唐国华', '刘国勇', '刘会连', '肖琥', '邱峰')" if except_senior else "" filter_department = "and department_name='{}'".format(department) if department else "" return ''' select department_name, @@ -34,9 +34,6 @@ group by department_name, user_name, procinst_id def querystring_senior_procinst_duration(start, end): return ''' -SELECT user_name, - ROUND(AVG(procinst_duration_in_minutes) /60, 2) as procinst_duration_in_hours_by_user -from ( select user_name, procinst_id, sum(duration_in_minutes) as procinst_duration_in_minutes @@ -56,11 +53,9 @@ from ( inner join workflow_process as wp on wpv.process_id=wp.id inner join user as u on wpa.deal_user_id=u.id where wpa.end_time >='{}' and wpa.end_time < '{}' and wp.deleted=false and wp.is_enable=true and u.delete=0 and u.state=1 and u.active_status=1 - and u.name in ('刘文峰', '金亮', '姜珍', '余莎莎', '张阳根', '唐国华', '刘国勇', '刘会连', '肖琥') + and u.name in ('刘文峰', '金亮', '姜珍', '余莎莎', '张阳根', '唐国华', '刘国勇', '刘会连', '肖琥', '邱峰') ) as r group by user_name, procinst_id -) as r2 -GROUP by r2.user_name '''.format(start, end) @@ -139,14 +134,22 @@ order by procinst_duration_in_hours_by_department DESC def querystring_procinst_duration_by_senior(start, end): return ''' SELECT ROUND(AVG(procinst_duration_in_hours_by_user), 2) as procinst_duration_in_hours_by_department -from ({}) as r3 +from ( + SELECT user_name, + ROUND(AVG(procinst_duration_in_minutes) /60, 2) as procinst_duration_in_hours_by_user + from ({}) as r2 + GROUP by r2.user_name +) as r3 '''.format(querystring_senior_procinst_duration(start, end)) # 用户单流程处理耗时（高管） def querystring_procinst_duration_by_user_senior(start, end): return ''' -{} +SELECT user_name, + ROUND(AVG(procinst_duration_in_minutes) /60, 2) as procinst_duration_in_hours_by_user +from ({}) as r2 +GROUP by r2.user_name order by procinst_duration_in_hours_by_user DESC '''.format(querystring_senior_procinst_duration(start, end)) @@ -154,7 +157,12 @@ order by procinst_duration_in_hours_by_user DESC # 各部门耗时较长用户（高管） def querystring_user_gt_senior_avg(start, end, avg): return ''' -SELECT * from ({}) as r3 +SELECT * from ( + SELECT user_name, + ROUND(AVG(procinst_duration_in_minutes) /60, 2) as procinst_duration_in_hours_by_user + from ({}) as r2 + GROUP by r2.user_name +) as r3 WHERE r3.procinst_duration_in_hours_by_user > {} order by procinst_duration_in_hours_by_user DESC '''.format(querystring_senior_procinst_duration(start, end), avg) @@ -178,8 +186,28 @@ order by procinst_duration_in_hours_by_user desc def querystring_procinst_duration_by_user(start, end, department): return """ SELECT user_name, - ROUND(AVG(procinst_duration_in_minutes) /60, 2) as procinst_duration_in_hours_by_user + ROUND(AVG(procinst_duration_in_minutes) /60, 2) as procinst_duration_in_hours_by_user from ({}) as r2 group by r2.user_name order by procinst_duration_in_hours_by_user DESC """.format(querystring_user_procinst_duration(start, end, True, department)) + + +# 按部门统计用户处理流程数量 +def querystring_procinst_count_by_user(start, end, department): + return ''' +SELECT user_name, + COUNT(*) as procinst_count_by_user +from ({}) as r2 +group by r2.user_name + '''.format(querystring_user_procinst_duration(start, end, True, department)) + + +# 用户处理流程数量（高管） +def querystring_procinst_count_by_senior(start, end): + return ''' +SELECT user_name, + COUNT(*) as procinst_count_by_user +from ({}) as r2 +group by r2.user_name + '''.format(querystring_senior_procinst_duration(start, end)) diff --git a/code/pep-stats-report/app/image_helper.py b/code/pep-stats-report/app/image_helper.py index e69de29..e62f6f5 100644 --- a/code/pep-stats-report/app/image_helper.py +++ b/code/pep-stats-report/app/image_helper.py @@ -0,0 +1,110 @@ +import platform +import os +import fitz # pip install PyMuPDF +from PIL import Image +import shutil +import logging + + +# 将word文件转换成pdf文件 +def word2pdf(word_file): + from win32com import client # pip install pywin32 + + # 获取word格式处理对象 + word = client.Dispatch('Word.Application') + # 以Doc对象打开文件 + doc_ = word.Documents.Open(word_file) + # 另存为pdf文件 + pdf_file = word_file.replace(os.path.basename(word_file).split('.')[1], "pdf") + doc_.SaveAs(pdf_file, FileFormat=17) + logging.info(f'{word_file} ----转pdf成功') + # 关闭doc对象 + doc_.Close() + # 退出word对象 + word.Quit() + return pdf_file + + +# 将word文件转换成pdf文件（Linux） +def word2pdf_linux(word_file): + word_path = os.path.dirname(word_file) + os.system(f"libreoffice --headless --language=zh-CN --convert-to pdf {word_file} --outdir {word_path}") + logging.info(f'{word_file} ----转pdf成功') + pdf_file = word_file.replace(os.path.basename(word_file).split('.')[1], "pdf") + return pdf_file + + +# pdf转图片 +def pdf2png(pdf_file): + image_path = os.path.abspath(f'{os.path.dirname(pdf_file)}/tmp_pdf2png') + try: + # 创建一个空白图片，用于拼接内容 + width, height = 0, 0 + images = [] + + pdf_doc = fitz.open(pdf_file) + for pg in range(pdf_doc.page_count): + page = pdf_doc[pg] + rotate = int(0) + # 每个尺寸的缩放系数为1.3，这将为我们生成分辨率提高2.6的图像。 + # 此处若是不做设置，默认图片大小为：792X612, dpi=96 + zoom_x = 1.33333333 # (1.33333333-->1056x816) (2-->1584x1224) + zoom_y = 1.33333333 + mat = fitz.Matrix(zoom_x, zoom_y).prerotate(rotate) + pix = page.get_pixmap(matrix=mat, alpha=False) + + if not os.path.exists(image_path): # 判断存放图片的文件夹是否存在 + os.makedirs(image_path) # 若图片文件夹不存在就创建 + pix.save(image_path + '/' + 'tmp%s.png' % pg) # 将图片写入指定的文件夹内 + + img = Image.open(image_path + '/' + 'tmp%s.png' % pg) + img_width, img_height = img.size + + # 更新拼接图片的宽度和高度 + width = max(width, img_width) + height += img_height + + # 添加图片到拼接列表 + images.append(img) + + # 创建一个空白长图 + long_image = Image.new('RGB', (width, height), (255, 255, 255)) + y_offset = 0 + + # 将每张图片拼接到长图中 + for img in images: + long_image.paste(img, (0, y_offset)) + y_offset += img.height + + # 保存拼接后的长图 + png_file = pdf_file.replace(os.path.basename(pdf_file).split('.')[1], "png") + long_image.save(png_file) + + # 删除中间临时保存的图片 + shutil.rmtree(image_path) + except IOError as error: + logging.error('pdf转png失败') + raise error + else: + logging.info("pdf转png成功") + return png_file + + +def word_to_long_image(word_file_path): + try: + _file = os.path.abspath(word_file_path) # os.path.abspath('input.docx') + + if platform.system().lower() == 'windows': + pdf_file = word2pdf(_file) + else: + pdf_file = word2pdf_linux(_file) + + png_file = pdf2png(pdf_file) + + # 删除中间保存的pdf文件 + os.remove(pdf_file) + + return png_file + except Exception as error: + logging.error('word转长图出错：{}'.format(error)) + raise error diff --git a/code/pep-stats-report/config.ini b/code/pep-stats-report/config.ini index 1439b05..9cda0fd 100644 --- a/code/pep-stats-report/config.ini +++ b/code/pep-stats-report/config.ini @@ -3,7 +3,7 @@ host = 10.8.30.161 port = 30900 username = default password = -database = pepca_m +database = pepca9 [qiniu] access-key=5XrM4wEB9YU6RQwT64sPzzE6cYFKZgssdP5Kj3uu diff --git a/code/pep-stats-report/main.py b/code/pep-stats-report/main.py index 67438b7..efacb36 100644 --- a/code/pep-stats-report/main.py +++ b/code/pep-stats-report/main.py @@ -7,7 +7,8 @@ from docx.shared import Inches, Pt, RGBColor from docx.enum.text import WD_PARAGRAPH_ALIGNMENT, WD_LINE_SPACING from docx.oxml.ns import qn import matplotlib.pyplot as plt -from app import db_helper +import numpy as np +from app import db_helper, image_helper import qiniu import logging @@ -107,6 +108,43 @@ def create_bar_chart(doc, x, y, mean, title, filename): r.add_picture('{}/{}'.format(REPORT_DIR, filename), width=Inches(6)) +def create_bar_twinx_chart(doc, x, y1, y2, mean, title, filename): + plt.rcParams["font.sans-serif"] = ["SimHei"] # 设置字体 + plt.rcParams["axes.unicode_minus"] = False # 正常显示负号 + + x_range = np.arange(len(x)) + + width = 0.25 # the width of the bars + + fig, ax1 = plt.subplots() + ax1.set_ylabel('耗时') + rects = ax1.bar(x_range, y1, width, color=(31 / 255, 168 / 255, 201 / 255), label='耗时') + ax1.bar_label(rects, padding=3) + ax1.set_xticks(x_range + width / 2, x) + # 设置x轴标签旋转角度 + plt.xticks(rotation=-30, ha='left') + # 绘制均值线 + plt.axhline(mean, linestyle='dashed', color='#FF8C00', label=f'均值：{mean}') + + ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis + + ax2.set_ylabel('流程数') + rects = ax2.bar(x_range + width, y2, width, color='#5AC189', label='流程数') + ax2.bar_label(rects, padding=3) + + plt.title(title) + fig.legend(loc='upper right', ncols=1, bbox_to_anchor=(1, 1), bbox_transform=ax1.transAxes) + fig.tight_layout() # otherwise the right y-label is slightly clipped + # 保存图形 + plt.savefig('{}/{}'.format(REPORT_DIR, filename)) + plt.close() + # 插入图形到 Word 文档中 + p = doc.add_paragraph() + p.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER + r = p.add_run() + r.add_picture('{}/{}'.format(REPORT_DIR, filename), width=Inches(6)) + + def add_section(doc, rank, department, d_elapse): h = doc.add_heading(level=2) r = h.add_run('{}. {}'.format(rank, department)) @@ -124,6 +162,12 @@ def add_section(doc, rank, department, d_elapse): ) if department == '高管' else db_helper.querystring_procinst_duration_by_user( start_time, end_time, department) procinst_duration_by_user = client.execute(qs) + + qs = db_helper.querystring_procinst_count_by_senior( + start_time, end_time + ) if department == '高管' else db_helper.querystring_procinst_count_by_user( + start_time, end_time, department) + procinst_count_by_user = client.execute(qs) except Exception as error: print('数据库查询错误：', error) logging.error('数据库查询错误：{}'.format(error)) @@ -149,11 +193,20 @@ def add_section(doc, rank, department, d_elapse): set_paragraph_format(p) x = [] - y = [] + y1 = [] + y2 = [] + for user, elapse in procinst_duration_by_user: x.append(user) - y.append(elapse) - create_bar_chart(doc, x, y, d_elapse, "个人流程处理平均耗时", "{}-流程处理平均耗时.png".format(department)) + y1.append(elapse) + + for name in x: + for user, count in procinst_count_by_user: + if user == name: + y2.append(count) + break + + create_bar_twinx_chart(doc, x, y1, y2, d_elapse, "个人流程处理平均耗时", "{}-流程处理平均耗时.png".format(department)) caption = doc.add_paragraph('图 2-{} {}个人处理流程平均耗时'.format(rank, department)) caption.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER @@ -193,7 +246,7 @@ def add_chapter_1(doc): logging.error('数据库查询错误：{}'.format(error)) raise RuntimeError('数据库查询错误：', error) - if procinst_duration_by_senior > procinst_duration_by_company: + if procinst_duration_by_senior is not None and procinst_duration_by_senior > procinst_duration_by_company: department_count_gt_avg += 1 p = doc.add_paragraph() @@ -289,12 +342,17 @@ def generate_word_report(): set_paragraph_format(p_last) # 保存文档 - report_path = '{}/项企流程效能分析结果公示（全员）-{}年{}月.docx'.format(REPORT_DIR, last_year, last_month) - doc.save(report_path) - logging.info('本地报表创建成功：{}'.format(report_path)) + report_file_name = '项企流程效能分析结果公示（全员）-{}年{}月.docx'.format(last_year, last_month) + report_file = f'{REPORT_DIR}/{report_file_name}' + doc.save(report_file) + logging.info('本地报表创建成功：{}'.format(report_file)) + + # 将文档转换为图片 + png_file = image_helper.word_to_long_image(report_file) # 上传七牛云 - upload_to_qiniu('项企流程效能分析结果公示（全员）-{}年{}月.docx'.format(last_year, last_month)) + upload_to_qiniu(report_file_name) + upload_to_qiniu(os.path.basename(png_file)) def create_clickhouse_client(): @@ -343,7 +401,7 @@ if __name__ == '__main__': procinst_duration_by_senior = client.execute(qs3)[0][0] for index, (department, elapse) in enumerate(procinst_duration_by_department): - if procinst_duration_by_senior >= elapse: + if procinst_duration_by_senior is not None and procinst_duration_by_senior >= elapse: procinst_duration_by_department.insert(index, ('高管', procinst_duration_by_senior)) break diff --git a/code/pep-stats-report/requirements.txt b/code/pep-stats-report/requirements.txt index f0f2820..e91d0da 100644 --- a/code/pep-stats-report/requirements.txt +++ b/code/pep-stats-report/requirements.txt @@ -19,11 +19,13 @@ pefile==2023.2.7 Pillow==9.5.0 pyinstaller==5.10.1 pyinstaller-hooks-contrib==2023.2 +PyMuPDF==1.22.3 pyparsing==3.0.9 python-dateutil==2.8.2 python-docx==0.8.11 pytz==2023.3 pytz-deprecation-shim==0.1.0.post0 +pywin32==306 pywin32-ctypes==0.2.0 qiniu==7.10.0 requests==2.28.2