From 8658f7f039e431f5fed5df67cf81241e12e8f0aa Mon Sep 17 00:00:00 2001 From: "gao.zhiyuan" Date: Wed, 29 Jun 2022 09:27:04 +0800 Subject: [PATCH] init --- .gitignore | 123 +++++++++++++++ code/api/.vscode/launch.json | 40 +++++ code/api/Dockerfile | 21 +++ code/api/app/index.js | 3 + code/api/app/lib/index.js | 27 ++++ code/api/app/lib/middlewares/api-log.js | 83 ++++++++++ code/api/app/lib/middlewares/authenticator.js | 143 ++++++++++++++++++ code/api/app/lib/routes/index.js | 17 +++ code/api/app/lib/schedule/index.js | 18 +++ code/api/app/lib/service/redis.js | 44 ++++++ code/api/config.js | 117 ++++++++++++++ code/api/package.json | 35 +++++ code/api/server.js | 12 ++ code/api/utils/forward-api.js | 15 ++ code/web | 1 + 15 files changed, 699 insertions(+) create mode 100644 .gitignore create mode 100644 code/api/.vscode/launch.json create mode 100644 code/api/Dockerfile create mode 100644 code/api/app/index.js create mode 100644 code/api/app/lib/index.js create mode 100644 code/api/app/lib/middlewares/api-log.js create mode 100644 code/api/app/lib/middlewares/authenticator.js create mode 100644 code/api/app/lib/routes/index.js create mode 100644 code/api/app/lib/schedule/index.js create mode 100644 code/api/app/lib/service/redis.js create mode 100644 code/api/config.js create mode 100644 code/api/package.json create mode 100644 code/api/server.js create mode 100644 code/api/utils/forward-api.js create mode 160000 code/web diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e687ee9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,123 @@ +# ---> Node +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.env.test +.env.production + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +*yarn.lock +*package-lock.json +*log/ diff --git a/code/api/.vscode/launch.json b/code/api/.vscode/launch.json new file mode 100644 index 0000000..7fe4b73 --- /dev/null +++ b/code/api/.vscode/launch.json @@ -0,0 +1,40 @@ +{ + // 使用 IntelliSense 了解相关属性。 + // 悬停以查看现有属性的描述。 + // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "启动API", + "program": "${workspaceRoot}/server.js", + "env": { + "NODE_ENV": "development" + }, + "args": [ + "-p 4200", + "-f http://localhost:4200", + "-g postgres://postgres:123@10.8.30.32:5432/xxxx", + "--redisHost 127.0.0.1", + "--redisPort 6379" + ] + }, + { + "type": "node", + "request": "launch", + "name": "run mocha", + "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha", + "stopOnEntry": false, + "args": [ + "app/test/*.test.js", + "--no-timeouts" + ], + "cwd": "${workspaceRoot}", + "runtimeExecutable": null, + "env": { + "NODE_ENV": "development" + } + } + ] +} \ No newline at end of file diff --git a/code/api/Dockerfile b/code/api/Dockerfile new file mode 100644 index 0000000..0b756f7 --- /dev/null +++ b/code/api/Dockerfile @@ -0,0 +1,21 @@ +FROM registry.cn-hangzhou.aliyuncs.com/fs-devops/node:12-dev as builder + +COPY . /var/app + +WORKDIR /var/app + +EXPOSE 8080 + +RUN npm config set registry=http://10.8.30.22:7000 +RUN echo "{\"time\":\"$BUILD_TIMESTAMP\",\"build\": \"$BUILD_NUMBER\",\"revision\": \"$SVN_REVISION_1\",\"URL\":\"$SVN_URL_1\"}" > version.json +RUN npm cache clean -f +RUN rm -rf package-lock.json +RUN npm install --registry http://10.8.30.22:7000 + +FROM registry.cn-hangzhou.aliyuncs.com/fs-devops/node:12 + +COPY --from=builder --chown=node /var/app /home/node/app + +WORKDIR /home/node/app + +CMD ["node", "server.js"] \ No newline at end of file diff --git a/code/api/app/index.js b/code/api/app/index.js new file mode 100644 index 0000000..e1436de --- /dev/null +++ b/code/api/app/index.js @@ -0,0 +1,3 @@ +'use strict'; + +module.exports = require('./lib'); \ No newline at end of file diff --git a/code/api/app/lib/index.js b/code/api/app/lib/index.js new file mode 100644 index 0000000..14d8b1a --- /dev/null +++ b/code/api/app/lib/index.js @@ -0,0 +1,27 @@ +'use strict'; + +const routes = require('./routes'); +const authenticator = require('./middlewares/authenticator'); +// const apiLog = require('./middlewares/api-log'); +const redisConnect = require('./service/redis') +const schedule = require('./schedule') + +module.exports.entry = function (app, router, opts) { + app.fs.logger.log('info', '[FS-AUTH]', 'Inject auth and api mv into router.'); + + app.fs.api = app.fs.api || {}; + app.fs.api.authAttr = app.fs.api.authAttr || {}; + app.fs.api.logAttr = app.fs.api.logAttr || {}; + + redisConnect(app, opts) + schedule(app, opts) + router.use(authenticator(app, opts)); + // router.use(apiLog(app, opts)); + + router = routes(app, router, opts); +}; + +module.exports.models = function (dc) { // dc = { orm: Sequelize对象, ORM: Sequelize, models: {} } + require('./models/user')(dc); + require('./models/user_token')(dc); +}; diff --git a/code/api/app/lib/middlewares/api-log.js b/code/api/app/lib/middlewares/api-log.js new file mode 100644 index 0000000..fb17f66 --- /dev/null +++ b/code/api/app/lib/middlewares/api-log.js @@ -0,0 +1,83 @@ +/** + * Created by PengPeng on 2017/4/26. + */ +'use strict'; + +const moment = require('moment'); +const pathToRegexp = require('path-to-regexp'); + +function factory(app, opts) { + async function sendToEsAsync(producer, payloads) { + return new Promise((resolve, reject) => { + producer.send(payloads, function (err) { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }) + } + + async function logger(ctx, next) { + const { path, method } = ctx; + const start = Date.now(); + + // 等待路由处理 + await next(); + + try { + let logAttr = null; + for (let prop in app.fs.api.logAttr) { + let keys = []; + let re = pathToRegexp(prop.replace(/\:[A-Za-z_\-]+\b/g, '(\\d+)'), keys); + if (re.test(`${method}${path}`)) { + logAttr = app.fs.api.logAttr[prop]; + break; + } + } + let parameter = null, parameterShow = null, user_id, _token, app_key; + if (ctx.fs.api) { + const { actionParameter, actionParameterShow, userId, token, appKey } = ctx.fs.api; + parameter = actionParameter; + parameterShow = actionParameterShow; + user_id = userId; + _token = token; + app_key = appKey; + } + const producer = ctx.fs.kafka.producer; + + const message = { + log_time: moment().toISOString(), + method: method, + content: logAttr ? logAttr.content : '', + parameter: JSON.stringify(parameter) || JSON.stringify(ctx.request.body), + parameter_show: parameterShow, + visible: logAttr ? logAttr.visible : true, + cost: Date.now() - start, + status_code: ctx.status, + url: ctx.request.url, + user_agent: ctx.request.headers["user-agent"], + user_id: user_id, + session: _token, + app_key: app_key, + header: JSON.stringify(ctx.request.headers), + ip: ctx.request.headers["x-real-ip"] || ctx.ip + }; + + const payloads = [{ + topic: `${opts.kafka.topicPrefix}`, + messages: [JSON.stringify(message)], + partition: 0 + }]; + + // await sendToEsAsync(producer, payloads); + + } catch (e) { + ctx.fs.logger.error(`日志记录失败: ${e}`); + } + } + return logger; +} + +module.exports = factory; diff --git a/code/api/app/lib/middlewares/authenticator.js b/code/api/app/lib/middlewares/authenticator.js new file mode 100644 index 0000000..69a6cef --- /dev/null +++ b/code/api/app/lib/middlewares/authenticator.js @@ -0,0 +1,143 @@ +/** + * Created by PengLing on 2017/3/27. + */ +'use strict'; + +const pathToRegexp = require('path-to-regexp'); +const util = require('util'); +const moment = require('moment'); + +class ExcludesUrls { + constructor(opts) { + this.allUrls = undefined; + this.reload(opts); + } + + sanitizePath (path) { + if (!path) return '/'; + const p = '/' + path.replace(/^\/+/i, '').replace(/\/+$/, '').replace(/\/{2,}/, '/'); + return p; + } + + reload (opts) { + // load all url + if (!this.allUrls) { + this.allUrls = opts; + let that = this; + this.allUrls.forEach(function (url, i, arr) { + if (typeof url === "string") { + url = { p: url, o: '*' }; + arr[i] = url; + } + const keys = []; + let eachPath = url.p; + url.p = (!eachPath || eachPath === '(.*)' || util.isRegExp(eachPath)) ? eachPath : that.sanitizePath(eachPath); + url.pregexp = pathToRegexp(eachPath, keys); + }); + } + } + + isExcluded (path, method) { + return this.allUrls.some(function (url) { + return !url.auth + && url.pregexp.test(path) + && (url.o === '*' || url.o.indexOf(method) !== -1); + }); + } +} + +/** + * 判断Url是否不鉴权 + * @param {*} opts {exclude: [*] or []}，'*'或['*']:跳过所有路由; []:所有路由都要验证 + * @param {*} path 当前request的path + * @param {*} method 当前request的method + */ +let isPathExcluded = function (opts, path, method) { + let excludeAll = Boolean(opts.exclude && opts.exclude.length && opts.exclude[0] == '*'); + let excludes = null; + if (!excludeAll) { + let excludeOpts = opts.exclude || []; + excludeOpts.push({ p: '/login', o: 'POST' }); + excludeOpts.push({ p: '/logout', o: 'PUT' }); + excludes = new ExcludesUrls(excludeOpts); + } + let excluded = excludeAll || excludes.isExcluded(path, method); + return excluded; +}; + +let authorizeToken = async function (ctx, token) { + let rslt = null; + const tokenFormatRegexp = /^(\{{0,1}([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}\}{0,1})$/g; + if (token && tokenFormatRegexp.test(token)) { + try { + const expired = await ctx.redis.hget(token, 'expired'); + if (expired && moment().valueOf() <= moment(expired).valueOf()) { + const userInfo = JSON.parse(await ctx.redis.hget(token, 'userInfo')); + rslt = { + 'authorized': userInfo.authorized, + 'resources': (userInfo || {}).resources || [], + }; + ctx.fs.api.userId = userInfo.id; + ctx.fs.api.userInfo = userInfo; + ctx.fs.api.token = token; + } + } catch (err) { + const { error } = err.response || {}; + ctx.fs.logger.log('[anxinyun]', '[AUTH] failed', (error || {}).message || `cannot GET /users/${token}`); + } + } + return rslt; +}; + +let isResourceAvailable = function (resources, options) { + let authCode = null; + // authorize user by authorization attribute + const { authAttr, method, path } = options; + for (let prop in authAttr) { + let keys = []; + let re = pathToRegexp(prop.replace(/\:[A-Za-z_\-]+\b/g, '(\\d+)'), keys); + if (re.test(`${method}${path}`)) { + authCode = authAttr[prop]; + break; + } + } + return !authCode || (resources || []).some(code => code === authCode); +}; + +function factory (app, opts) { + return async function auth (ctx, next) { + const { path, method, header, query } = ctx; + ctx.fs.logger.log('[AUTH] start', path, method); + ctx.fs.api = ctx.fs.api || {}; + ctx.fs.port = opts.port; + ctx.redis = app.redis; + ctx.redisTools = app.redisTools; + let error = null; + if (path) { + if (!isPathExcluded(opts, path, method)) { + const user = await authorizeToken(ctx, header.token || query.token); + if (user && user.authorized) { + // if (!isResourceAvailable(user.resources, { authAttr: app.fs.auth.authAttr, path, method })) { + // error = { status: 403, name: 'Forbidden' } + // } else { + // error = { status: 401, name: 'Unauthorized' } + // } + } else { + error = { status: 401, name: 'Unauthorized' } + } + } + } else { + error = { status: 401, name: 'Unauthorized' }; + } + if (error) { + ctx.fs.logger.log('[AUTH] failed', path, method); + ctx.status = error.status; + ctx.body = error.name; + } else { + ctx.fs.logger.log('[AUTH] passed', path, method); + await next(); + } + } +} + +module.exports = factory; diff --git a/code/api/app/lib/routes/index.js b/code/api/app/lib/routes/index.js new file mode 100644 index 0000000..2d6a9f8 --- /dev/null +++ b/code/api/app/lib/routes/index.js @@ -0,0 +1,17 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); + +module.exports = function (app, router, opts) { + fs.readdirSync(__dirname).forEach((filename) => { + if (filename.indexOf('.') !== 0 &&fs.lstatSync(path.join(__dirname, filename)).isDirectory()) { + fs.readdirSync(path.join(__dirname, filename)).forEach((api) => { + if (api.indexOf('.') == 0 || api.indexOf('.js') == -1) return; + require(`./${filename}/${api}`)(app, router, opts); + }); + } + }); + + return router; +}; diff --git a/code/api/app/lib/schedule/index.js b/code/api/app/lib/schedule/index.js new file mode 100644 index 0000000..6681efa --- /dev/null +++ b/code/api/app/lib/schedule/index.js @@ -0,0 +1,18 @@ +'use strict'; + +const fs = require('fs'); +// 将定时任务汇集未来可根据需要选取操作 +module.exports = async function (app, opts) { + fs.readdirSync(__dirname).forEach((filename) => { + if (!['index.js'].some(f => filename == f)) { + const schedule = require(`./${filename}`)(app, opts) + for (let k of Object.keys(schedule)) { + console.info(`定时任务 ${k} 启动`); + } + app.fs.schedule = { + ...app.fs.schedule, + ...schedule, + } + } + }); +}; diff --git a/code/api/app/lib/service/redis.js b/code/api/app/lib/service/redis.js new file mode 100644 index 0000000..7e87933 --- /dev/null +++ b/code/api/app/lib/service/redis.js @@ -0,0 +1,44 @@ +'use strict'; +const redis = require("ioredis") +const moment = require('moment') + +module.exports = async function factory (app, opts) { + let client = new redis(opts.redis.port, opts.redis.host); + + client.on("error", function (err) { + app.fs.logger.error('info', '[FS-AUTH-REDIS]', 'redis connect error.'); + console.error("Error :", err); + process.exit(-1); + }); + + client.on('connect', function () { + console.log(`redis connect success ${opts.redis.host + ':' + opts.redis.port}`); + }) + + // 查询尚未过期token放入redis + // const tokenRes = await app.fs.dc.models.UserToken.findAll({ + // where: { + // expired: { $gte: moment().format('YYYY-MM-DD HH:mm:ss') } + // } + // }); + + // for (let t of tokenRes) { + // const { token, dataValues } = t + // dataValues.userInfo = JSON.stringify(dataValues.userInfo) + // dataValues.expired = moment(dataValues.expired).format() + // await client.hmset(token, dataValues); + // } + // token 2 redis end + + // 自定义方法 + async function hdelall (key) { + const obj = await client.hgetall(key); + const hkeys = Object.keys(obj) + await client.hdel(key, hkeys) + } + + app.redis = client + app.redisTools = { + hdelall, + } +} diff --git a/code/api/config.js b/code/api/config.js new file mode 100644 index 0000000..10242f3 --- /dev/null +++ b/code/api/config.js @@ -0,0 +1,117 @@ +'use strict'; +/*jslint node:true*/ +const path = require('path'); +const os = require('os'); +const moment = require('moment'); +const args = require('args'); + +const dev = process.env.NODE_ENV == 'development'; + +// 启动参数 +args.option(['p', 'port'], '启动端口'); +args.option(['g', 'pg'], 'postgre服务URL'); +args.option(['f', 'fileHost'], '文件中心本地化存储: WebApi 服务器地址(必填), 该服务器提供文件上传Web服务'); +args.option('redisHost', 'redisHost'); +args.option('redisPort', 'redisPort'); +args.option('redisPswd', 'redisPassword'); + +const flags = args.parse(process.argv); + +const IOT_AUTH_DB = process.env.IOT_AUTH_DB || flags.pg; +const IOT_AUTH_LOCAL_SVR_ORIGIN = process.env.IOT_AUTH_LOCAL_SVR_ORIGIN || flags.fileHost; + +const IOTA_REDIS_SERVER_HOST = process.env.IOTA_REDIS_SERVER_HOST || flags.redisHost || "localhost";//redis IP +const IOTA_REDIS_SERVER_PORT = process.env.IOTA_REDIS_SERVER_PORT || flags.redisPort || "6379";//redis 端口 +const IOTA_REDIS_SERVER_PWD = process.env.IOTA_REDIS_SERVER_PWD || flags.redisPswd || "";//redis 密码 + +if (!IOT_AUTH_DB || !IOTA_REDIS_SERVER_HOST || !IOTA_REDIS_SERVER_PORT) { + console.log('缺少启动参数，异常退出'); + args.showHelp(); + process.exit(-1); +} + +const product = { + port: flags.port || 8080, + staticDirs: ['static'], + mws: [ + { + entry: require('@fs/attachment').entry, + opts: { + local: { + origin: IOT_AUTH_LOCAL_SVR_ORIGIN || `http://localhost:${flags.port || 8080}`, + rootPath: 'static', + childPath: 'upload', + }, + maxSize: 104857600, // 100M + } + }, { + entry: require('./app').entry, + opts: { + exclude: [// 不做认证的路由，也可以使用 exclude: ["*"] 跳过所有路由 + // { p: '/cross_token/check', o: 'POST' } + ], + redis: { + host: IOTA_REDIS_SERVER_HOST, + port: IOTA_REDIS_SERVER_PORT, + pwd: IOTA_REDIS_SERVER_PWD + }, + } + } + ], + dc: { + url: IOT_AUTH_DB, + opts: { + pool: { + max: 80, + min: 10, + idle: 10000 + }, + define: { + freezeTableName: true, // 固定表名 + timestamps: false // 不含列 "createAt"/"updateAt"/"DeleteAt" + }, + timezone: '+08:00', + logging: false + }, + models: [require('./app').models] + }, + logger: { + level: 'info', + json: false, + filename: path.join(__dirname, 'log', 'runtime.log'), + colorize: false, + maxsize: 1024 * 1024 * 5, + rotationFormat: false, + zippedArchive: true, + maxFiles: 10, + prettyPrint: true, + label: '', + timestamp: () => moment().format('YYYY-MM-DD HH:mm:ss.SSS'), + eol: os.EOL, + tailable: true, + depth: null, + showLevel: true, + maxRetries: 1 + } +}; + +const development = { + port: product.port, + staticDirs: product.staticDirs, + mws: product.mws, + dc: product.dc, + logger: product.logger +}; + +if (dev) { + // mws + for (let mw of development.mws) { + // if (mw.opts.exclude) mw.opts.exclude = ['*']; // 使用 ['*'] 跳过所有路由 + } + // logger + development.logger.filename = path.join(__dirname, 'log', 'development.log'); + development.logger.level = 'debug'; + development.dc.opts.logging = console.log; +} + +module.exports = dev ? development : product; diff --git a/code/api/package.json b/code/api/package.json new file mode 100644 index 0000000..ac8bdd3 --- /dev/null +++ b/code/api/package.json @@ -0,0 +1,35 @@ +{ + "name": "iot-auth", + "version": "1.0.0", + "description": "fs iot-auth api", + "main": "server.js", + "scripts": { + "test": "set DEBUG=true&&\"node_modules/.bin/mocha\" --harmony --reporter spec app/test/*.test.js", + "start": "set NODE_ENV=development&&node server -p 4200 -g postgres://postgres:123@10.8.30.32:5432/iot_auth -f http://localhost:4200", + "start:linux": "export NODE_ENV=development&&node server -p 4200 -g postgres://postgres:123@10.8.30.32:5432/iot_auth" + }, + "author": "", + "license": "MIT", + "repository": {}, + "dependencies": { + "@fs/attachment": "^1.0.0", + "args": "^3.0.7", + "crypto-js": "^4.0.0", + "file-saver": "^2.0.2", + "fs-web-server-scaffold": "^2.0.2", + "ioredis": "^5.0.4", + "koa-convert": "^1.2.0", + "koa-proxy": "^0.9.0", + "moment": "^2.24.0", + "node-schedule": "^2.1.0", + "path": "^0.12.7", + "path-to-regexp": "^3.0.0", + "pg": "^7.9.0", + "request": "^2.88.2", + "superagent": "^3.5.2", + "uuid": "^3.3.2" + }, + "devDependencies": { + "mocha": "^6.0.2" + } +} diff --git a/code/api/server.js b/code/api/server.js new file mode 100644 index 0000000..9d1454d --- /dev/null +++ b/code/api/server.js @@ -0,0 +1,12 @@ +/** + * Created by rain on 2016/1/25. + */ + +'use strict'; +/*jslint node:true*/ +//from koa + +const scaffold = require('fs-web-server-scaffold'); +const config = require('./config'); + +module.exports = scaffold(config); \ No newline at end of file diff --git a/code/api/utils/forward-api.js b/code/api/utils/forward-api.js new file mode 100644 index 0000000..6b48e3e --- /dev/null +++ b/code/api/utils/forward-api.js @@ -0,0 +1,15 @@ +'use strict'; +const proxy = require('koa-proxy'); +const convert = require('koa-convert'); + +module.exports = { + entry: function (app, router, opts) { + app.use(convert(proxy({ + host: opts.host, + match: opts.match, + map: function (path) { + return path.replace(opts.match, ''); + } + }))); + } +}; \ No newline at end of file diff --git a/code/web b/code/web new file mode 160000 index 0000000..f40cb38 --- /dev/null +++ b/code/web @@ -0,0 +1 @@ +Subproject commit f40cb384a3b63063971ee416c97f78e65e86ecf1