Browse Source

数据脚本

release_0.0.1
巴林闲侠 3 years ago
parent
commit
3dafc5de6a
  1. 3
      api/app/lib/controllers/data/road.js
  2. 9
      api/log/development.log
  3. 107
      scripts/0.0.1/data/工具脚本(无需执行)/dataIn.js
  4. 16
      scripts/0.0.1/data/工具脚本(无需执行)/index.js
  5. 50
      scripts/0.0.1/data/工具脚本(无需执行)/道路_数据库表对应.json

3
api/app/lib/controllers/data/road.js

@ -32,5 +32,6 @@ async function importIn (ctx) {
}
module.exports = {
importIn
importIn,
};

9
api/log/development.log

@ -6954,3 +6954,12 @@
2022-07-20 21:00:55.063 - debug: [FS-LOGGER] Init.
2022-07-20 21:00:55.128 - info: [FS-ATTACHMENT] Inject attachment mw into router.
2022-07-20 21:00:55.129 - info: [FS-AUTH] Inject auth and api mv into router.
2022-07-21 09:15:39.313 - debug: [FS-LOGGER] Init.
2022-07-21 09:15:39.402 - info: [FS-ATTACHMENT] Inject attachment mw into router.
2022-07-21 09:15:39.402 - info: [FS-AUTH] Inject auth and api mv into router.
2022-07-21 09:27:02.043 - error: path: /department/user, error: SequelizeValidationError: notNull Violation: user.username cannot be null
2022-07-21 09:33:35.924 - error: path: /department/user/1, error: SequelizeValidationError: notNull Violation: user.username cannot be null
2022-07-21 09:33:37.379 - error: path: /department/user/1, error: SequelizeValidationError: notNull Violation: user.username cannot be null
2022-07-21 09:33:38.086 - error: path: /department/user/1, error: SequelizeValidationError: notNull Violation: user.username cannot be null
2022-07-21 09:38:23.005 - error: path: /department/user, error: SequelizeValidationError: notNull Violation: user.username cannot be null
2022-07-21 09:42:39.489 - error: path: /department/user, error: SequelizeValidationError: notNull Violation: user.username cannot be null

107
scripts/0.0.1/data/工具脚本(无需执行)/dataIn.js

@ -0,0 +1,107 @@
try {
const { Pool, Client } = require('pg')
const request = require('superagent');
const Hex = require('crypto-js/enc-hex');
const MD5 = require('crypto-js/md5');
const XLSX = require('xlsx')
const path = require('path')
const fs = require("fs");
// 连接数据库
const pool = new Pool({
user: 'FashionAdmin',
host: '10.8.30.156',
database: 'SmartEmergency',
password: '123456',
port: 5432,
})
let appid = '20200917000567738';
let key = 'xXm4jsuuD38JIkkhEcK6';
const getAnswer = async (query) => {
let start = (new Date()).getTime();
let salt = start;
let str1 = appid + query + salt + key;
let sign = Hex.stringify(MD5(str1));
console.log(`翻译:${query}`);
let answer = await request.get('http://api.fanyi.baidu.com/api/trans/vip/translate').timeout(1000 * 30).query({
q: query,
appid: appid,
salt: salt,
from: 'zh',
to: 'en',
sign: sign
});
if (answer.body.error_code) {
console.warn(answer.body);
throw '百度不给力,快快debug'
}
let rslt = answer.body.trans_result;
// let upperCaseRslt = rslt[0].dst.toLowerCase().replace(/( |^)[a-z]/g, (L) => L.toUpperCase()).replace(/ /g, '');
// let upperCaseRslt = rslt[0].dst.toUpperCase().replace(/ /g, '_');
// let upperCaseRslt = rslt[0].dst.toLowerCase().replace(/ /g, '_');
let upperCaseRslt = rslt[0].dst.replace(/\//g, ' ').replace(/'/g, '').replace(/\s{2,}/g, '');
console.log(`翻译结果:${upperCaseRslt}`);
while (((new Date()).getTime() - start) < (1000 / 8)) {//每s只能调用10次
continue;
}
return upperCaseRslt
}
const fun = async () => {
// note: we don't try/catch this because if connecting throws an exception
// we don't need to dispose of the client (it will be undefined)
const client = await pool.connect()
try {
await client.query('BEGIN')
const fileList = [
{
path: './data/道路/乡道第三方.xls',
n: '道路',
tableName: 'road'
},
]
for (let f of fileList) {
console.log(`读取 ${f.path}`);
// 读取数据文件
let workbook = XLSX.readFile(path.join(__dirname, f.path))
let firstSheetName = workbook.SheetNames[0];
let worksheet = workbook.Sheets[firstSheetName];
let res = XLSX.utils.sheet_to_json(worksheet);
console.log(res[0]);
const keyMap = require(`./${f.n}_字段对应.json`);
console.log(keyMap);
for (let d of res) {
let data = {};
for (let k in keyMap) {
let v = d[keyMap[k]];
if (v) {
if (k == 'name') {
v = await getAnswer(v);
}
data[k] = v;
}
}
console.log(data);
await client.query(`INSERT INTO ${f.tableName} SET ?`, data);
}
}
// await client.query('ROLLBACK')
await client.query('COMMIT')
console.log('执行完毕~')
} catch (e) {
await client.query('ROLLBACK')
console.log('执行错误~')
throw e
} finally {
client.release();
}
}
fun()
} catch (error) {
console.error(error)
}

16
scripts/0.0.1/data/工具脚本(无需执行)/index.js

@ -56,11 +56,11 @@ try {
await client.query('BEGIN')
const fileList = [
// {
// path: './data/道路/乡道第三方.xls',
// n: '道路',
// tableName: 'road'
// },
{
path: './data/道路/乡道第三方.xls',
n: '道路',
tableName: 'road'
},
]
for (let f of fileList) {
@ -73,6 +73,7 @@ try {
console.log(res[0]);
let dataEx = res[0];
transResult = ''
sqlResult = ''
sql = `
-- ${f.n}
create table if not exists "${f.tableName}"
@ -99,13 +100,16 @@ try {
.replace(/( |^)[A-Z]/g, (L) => L.toLowerCase())
}", \n
`
sqlResult += `"${t}" : "${engT.trim().replace(/ /g, '_').replace(/( |^)[A-Z]/g, (L) => L.toLowerCase())}", \n
`
sql += `
alter table ${f.tableName} add ${upperEngT} varchar(1024);
comment on column ${f.tableName}.${upperEngT} is '${t}';
`
}
fs.writeFileSync(`${f.n}_字段对应.json`, `{${transResult}}`, 'utf-8');
// fs.writeFileSync(`${f.n}_字段对应.json`, `{${transResult}}`, 'utf-8');
// fs.writeFileSync(`${f.n}_数据脚本对应.sql`, sql, 'utf-8');
fs.writeFileSync(`${f.n}_数据库表对应.json`, `{${sqlResult}}`, 'utf-8');
}
// await client.query('ROLLBACK')

50
scripts/0.0.1/data/工具脚本(无需执行)/道路_数据库表对应.json

@ -0,0 +1,50 @@
{
"路线名称": "route_name",
"路线代码": "route_code",
"路段序号": "section_No",
"乡镇编码": "township_Code",
"起点地名": "starting_place_name",
"起点桩号": "start_station",
"起点分界点类别": "category_of_starting_point_and_dividing_point",
"止点地名": "stop_place_name",
"止点分界点类别": "category_of_dead_center_and_dividing_point",
"止点桩号": "stop_station",
"路段类型": "section_type",
"路段性质": "nature_of_road_section",
"建成时间": "completion_time",
"GBM及文明样板路": "gBM_and_civilized_model_road",
"地貌": "landforms",
"收费性质": "nature_of_charges",
"涵洞数量": "number_of_culverts",
"技术等级": "technical_level",
"路面类型": "pavement_type",
"路面宽度": "pavement_width",
"路基宽度": "subgrade_width",
"车道特征": "lane_characteristics",
"是否晴雨通车": "whether_it_is_open_to_traffic_in_sunny_or_rainy_days",
"设计时速": "design_speed_per_hour",
"是否城管路段": "urban_management_section_or_not",
"管养单位": "management_and_maintenance_unit",
"路政管理单位": "road_administration_unit",
"列养情况": "alimentation",
"列养资金来源": "source_of_listed_maintenance_funds",
"养护时间": "curing_time",
"可绿化里程": "greening_mileage",
"已绿化里程": "greening_mileaged",
"重复道路路段类型": "type_of_repeated_road_section",
"重复路段序号": "serial_number_of_repeated_section",
"重复路段路线编码": "repeated_section_route_code",
"填报单位": "reporting_unit",
"变更原因": "reason_for_change",
"变更时间": "change_time",
"是否按干线公路管理接养": "whether_maintenance_managed_highway",
"备注": "remarks",
"上年路线编码": "route_code_of_last_year",
"上年路线名称": "route_name_of_last_year",
"上年起点桩号": "starting_station_of_last_year",
"上年止点桩号": "last_years_ending_point_stake_number",
"图形里程": "graphic_mileage",
"桩号里程": "chainage_mileage",
"所在区县": "districtcounty",
"所在地市": "location_city"
}
Loading…
Cancel
Save