You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
285 lines
9.0 KiB
285 lines
9.0 KiB
'use strict';
|
|
|
|
const superagent = require('superagent');
|
|
const qiniu = require('qiniu');
|
|
const mammoth = require('mammoth');
|
|
const cheerio = require('cheerio');
|
|
const path = require('path');
|
|
const { randomUUID } = require('crypto');
|
|
|
|
const {
|
|
DOCX_IMPORT_MAX_FILE_SIZE,
|
|
DOCX_IMPORT_RESPONSE_TIMEOUT,
|
|
DOCX_IMPORT_DEADLINE_TIMEOUT,
|
|
DOCX_ZIP_MAGIC,
|
|
DOCX_CONTENT_TYPE_PATTERN,
|
|
} = require('./constants');
|
|
|
|
const ensureHttpUrl = (value, label = 'sourceFileUrl') => {
|
|
const raw = String(value || '').trim();
|
|
if (!raw) throw `missing param: ${label}`;
|
|
let parsed = null;
|
|
try {
|
|
parsed = new URL(raw);
|
|
} catch (error) {
|
|
throw `invalid param: ${label}`;
|
|
}
|
|
if (!['http:', 'https:'].includes(parsed.protocol)) {
|
|
throw `invalid param: ${label}`;
|
|
}
|
|
return parsed.toString();
|
|
};
|
|
|
|
const decodeURIComponentSafely = value => {
|
|
try {
|
|
return decodeURIComponent(value);
|
|
} catch (error) {
|
|
return value;
|
|
}
|
|
};
|
|
|
|
const parseDispositionFilename = (disposition = '') => {
|
|
const utf8Match = disposition.match(/filename\*\s*=\s*UTF-8''([^;]+)/i);
|
|
if (utf8Match && utf8Match[1]) {
|
|
return decodeURIComponentSafely(utf8Match[1].trim().replace(/^"|"$/g, ''));
|
|
}
|
|
const plainMatch = disposition.match(/filename\s*=\s*("?)([^";]+)\1/i);
|
|
if (plainMatch && plainMatch[2]) return plainMatch[2].trim();
|
|
return '';
|
|
};
|
|
|
|
const getSourceFileName = (sourceFileUrl, contentDisposition = '') => {
|
|
const fromDisposition = parseDispositionFilename(String(contentDisposition || ''));
|
|
if (fromDisposition) return fromDisposition;
|
|
try {
|
|
const parsed = new URL(sourceFileUrl);
|
|
const base = path.basename(parsed.pathname || '');
|
|
if (base) return decodeURIComponentSafely(base);
|
|
} catch (error) { }
|
|
return 'import.docx';
|
|
};
|
|
|
|
const toBuffer = data => {
|
|
if (!data) return Buffer.alloc(0);
|
|
if (Buffer.isBuffer(data)) return data;
|
|
if (data instanceof ArrayBuffer) return Buffer.from(data);
|
|
if (ArrayBuffer.isView(data)) return Buffer.from(data.buffer, data.byteOffset, data.byteLength);
|
|
if (typeof data === 'string') return Buffer.from(data);
|
|
return Buffer.alloc(0);
|
|
};
|
|
|
|
const hasDocxZipMagic = buffer => {
|
|
if (!Buffer.isBuffer(buffer) || buffer.length < DOCX_ZIP_MAGIC.length) return false;
|
|
return DOCX_ZIP_MAGIC.every((byte, index) => buffer[index] === byte);
|
|
};
|
|
|
|
const ensureDocxFile = ({ sourceFileUrl, contentType, fileBuffer }) => {
|
|
const safeContentType = String(contentType || '').toLowerCase();
|
|
const hasDocxMime = DOCX_CONTENT_TYPE_PATTERN.test(safeContentType);
|
|
const pathname = (() => {
|
|
try {
|
|
return new URL(sourceFileUrl).pathname || '';
|
|
} catch (error) {
|
|
return '';
|
|
}
|
|
})();
|
|
const hasDocxExt = pathname.toLowerCase().endsWith('.docx');
|
|
if (!hasDocxZipMagic(fileBuffer)) {
|
|
throw 'source file is not a valid docx file';
|
|
}
|
|
if (!hasDocxExt && !hasDocxMime) {
|
|
throw 'source file must be a docx file';
|
|
}
|
|
};
|
|
|
|
const downloadDocxFile = async sourceFileUrl => {
|
|
const response = await superagent
|
|
.get(sourceFileUrl)
|
|
.redirects(3)
|
|
.responseType('arraybuffer')
|
|
.timeout({
|
|
response: DOCX_IMPORT_RESPONSE_TIMEOUT,
|
|
deadline: DOCX_IMPORT_DEADLINE_TIMEOUT,
|
|
});
|
|
|
|
const fileBuffer = toBuffer(response.body);
|
|
if (!fileBuffer.length) throw 'source file is empty';
|
|
if (fileBuffer.length > DOCX_IMPORT_MAX_FILE_SIZE) {
|
|
throw `source file too large, max ${Math.floor(DOCX_IMPORT_MAX_FILE_SIZE / 1024 / 1024)}MB`;
|
|
}
|
|
|
|
const contentType = String(response.header?.['content-type'] || '').trim();
|
|
const sourceFileName = getSourceFileName(
|
|
sourceFileUrl,
|
|
String(response.header?.['content-disposition'] || '')
|
|
);
|
|
|
|
ensureDocxFile({
|
|
sourceFileUrl,
|
|
contentType,
|
|
fileBuffer,
|
|
});
|
|
|
|
return {
|
|
fileBuffer,
|
|
contentType,
|
|
sourceFileName,
|
|
};
|
|
};
|
|
|
|
const ensureQiniuConfig = ctx => {
|
|
const qiniuConfig = ctx.config?.qiniu || {};
|
|
const bucket = String(qiniuConfig.bkt || '').trim();
|
|
const accessKey = String(qiniuConfig.ak || '').trim();
|
|
const secretKey = String(qiniuConfig.sk || '').trim();
|
|
const domain = String(qiniuConfig.dmn || '').trim().replace(/\/+$/, '');
|
|
if (!bucket || !accessKey || !secretKey) {
|
|
throw 'qiniu config is incomplete';
|
|
}
|
|
return {
|
|
bucket,
|
|
accessKey,
|
|
secretKey,
|
|
domain,
|
|
};
|
|
};
|
|
|
|
const createQiniuUploader = ctx => {
|
|
const { bucket, accessKey, secretKey, domain } = ensureQiniuConfig(ctx);
|
|
const mac = new qiniu.auth.digest.Mac(accessKey, secretKey);
|
|
const putPolicy = new qiniu.rs.PutPolicy({ scope: bucket });
|
|
const uploadToken = putPolicy.uploadToken(mac);
|
|
const config = new qiniu.conf.Config();
|
|
config.zone = qiniu.zone.Zone_z0;
|
|
const formUploader = new qiniu.form_up.FormUploader(config);
|
|
|
|
return async (key, buffer, mimeType = 'application/octet-stream') => {
|
|
const putExtra = new qiniu.form_up.PutExtra();
|
|
putExtra.mimeType = mimeType;
|
|
const uploadBody = await new Promise((resolve, reject) => {
|
|
formUploader.put(uploadToken, key, buffer, putExtra, (respErr, respBody, respInfo) => {
|
|
if (respErr) {
|
|
reject(respErr);
|
|
return;
|
|
}
|
|
if (respInfo?.statusCode && respInfo.statusCode >= 300) {
|
|
reject(new Error(`qiniu upload failed with status ${respInfo.statusCode}`));
|
|
return;
|
|
}
|
|
resolve(respBody || {});
|
|
});
|
|
});
|
|
const finalKey = String(uploadBody.key || key);
|
|
return domain ? `${domain}/${finalKey}` : finalKey;
|
|
};
|
|
};
|
|
|
|
const getImageExtensionByContentType = contentType => {
|
|
const normalized = String(contentType || '').toLowerCase();
|
|
const map = {
|
|
'image/png': 'png',
|
|
'image/jpeg': 'jpg',
|
|
'image/jpg': 'jpg',
|
|
'image/webp': 'webp',
|
|
'image/gif': 'gif',
|
|
'image/bmp': 'bmp',
|
|
'image/tiff': 'tiff',
|
|
'image/svg+xml': 'svg',
|
|
};
|
|
return map[normalized] || 'bin';
|
|
};
|
|
|
|
const buildDocxImageQiniuKey = contentType => {
|
|
const extension = getImageExtensionByContentType(contentType);
|
|
const datePart = new Date().toISOString().slice(0, 10).replace(/-/g, '');
|
|
return `report/docx-import/images/${datePart}/${randomUUID()}.${extension}`;
|
|
};
|
|
|
|
const convertDocxToHtml = async (ctx, fileBuffer) => {
|
|
let uploadToQiniu = null;
|
|
const result = await mammoth.convertToHtml(
|
|
{ buffer: fileBuffer },
|
|
{
|
|
convertImage: mammoth.images.imgElement(async image => {
|
|
const contentType = String(image.contentType || 'application/octet-stream');
|
|
const imageBuffer = toBuffer(await image.read());
|
|
if (!imageBuffer.length) throw new Error('empty image in docx');
|
|
if (!uploadToQiniu) {
|
|
uploadToQiniu = createQiniuUploader(ctx);
|
|
}
|
|
const imageKey = buildDocxImageQiniuKey(contentType);
|
|
const src = await uploadToQiniu(imageKey, imageBuffer, contentType);
|
|
return { src };
|
|
})
|
|
}
|
|
);
|
|
return String(result?.value || '').trim();
|
|
};
|
|
|
|
const parseDocxHtmlToChapters = html => {
|
|
const $ = cheerio.load(`<div id="docx-import-root">${String(html || '')}</div>`, { decodeEntities: false });
|
|
const root = $('#docx-import-root');
|
|
const chapters = [];
|
|
const stack = [];
|
|
const pendingBeforeFirstHeading = [];
|
|
|
|
root.contents().each((_, element) => {
|
|
if (element.type === 'text' && !String(element.data || '').trim()) return;
|
|
const serialized = String($.html(element) || '').trim();
|
|
if (!serialized) return;
|
|
|
|
const tagName = String(element.name || element.tagName || '').toLowerCase();
|
|
const headingMatch = tagName.match(/^h([1-6])$/);
|
|
if (!headingMatch) {
|
|
if (!stack.length) {
|
|
pendingBeforeFirstHeading.push(serialized);
|
|
return;
|
|
}
|
|
stack[stack.length - 1].contentParts.push(serialized);
|
|
return;
|
|
}
|
|
|
|
const level = Number(headingMatch[1]);
|
|
const title = String($(element).text() || '').replace(/\s+/g, ' ').trim();
|
|
const node = {
|
|
title: title || 'Untitled Chapter',
|
|
level,
|
|
contentParts: [],
|
|
children: [],
|
|
};
|
|
|
|
if (!stack.length && pendingBeforeFirstHeading.length) {
|
|
node.contentParts.push(...pendingBeforeFirstHeading);
|
|
pendingBeforeFirstHeading.length = 0;
|
|
}
|
|
|
|
while (stack.length && stack[stack.length - 1].level >= level) {
|
|
stack.pop();
|
|
}
|
|
if (stack.length) {
|
|
stack[stack.length - 1].children.push(node);
|
|
} else {
|
|
chapters.push(node);
|
|
}
|
|
stack.push(node);
|
|
});
|
|
|
|
if (!chapters.length) {
|
|
throw 'docx 文档未识别到标题，请使用 heading 样式';
|
|
}
|
|
|
|
const normalizeNode = node => ({
|
|
title: node.title,
|
|
contentHtml: node.contentParts.join('\n').trim(),
|
|
children: (node.children || []).map(normalizeNode),
|
|
});
|
|
|
|
return chapters.map(normalizeNode);
|
|
};
|
|
|
|
module.exports = {
|
|
ensureHttpUrl,
|
|
downloadDocxFile,
|
|
convertDocxToHtml,
|
|
parseDocxHtmlToChapters,
|
|
};
|
|
|