From 9fbf7cc0c73b706d7165645e4d05ab64e8747f27 Mon Sep 17 00:00:00 2001 From: ssube Date: Sat, 2 Nov 2019 09:03:41 -0500 Subject: [PATCH] feat: load rules from directories (#8) and modules (#6) --- src/config/args.ts | 45 ++++++++++++++++++------ src/index.ts | 28 ++++++--------- src/rule/index.ts | 87 +++++++++++++++++++++++++++++++++++++++++++--- src/source.ts | 4 ++- src/utils/index.ts | 2 +- 5 files changed, 131 insertions(+), 35 deletions(-) diff --git a/src/config/args.ts b/src/config/args.ts index be9ef45..55c83fc 100644 --- a/src/config/args.ts +++ b/src/config/args.ts @@ -1,8 +1,17 @@ import { Options, showCompletionScript, usage } from 'yargs'; -import { RuleSelector } from '../rule'; +import { RuleSelector, RuleSources } from '../rule'; import { VERSION_INFO } from '../version'; +export enum MODE { + check = 'check', + complete = 'complete', + fix = 'fix', + list = 'list', +} + +export const VALID_MODES = new Set([MODE.check, MODE.fix, MODE.list]); + /* tslint:disable:no-any */ export const CONFIG_ARGS_NAME = 'config-name'; @@ -19,7 +28,7 @@ export interface Args { mode: string; } -export interface ParsedArgs extends RuleSelector { +export interface ParsedArgs extends RuleSelector, RuleSources { [CONFIG_ARGS_NAME]: string; [CONFIG_ARGS_PATH]: string; coerce: boolean; @@ -33,7 +42,7 @@ export interface ParsedArgs extends RuleSelector { export interface ParseResults { args: ParsedArgs; - mode: string; + mode: MODE; } /** @@ -42,14 +51,14 @@ export interface ParseResults { * @TODO: fix it to use argv, not sure if yargs can do that */ export function parseArgs(argv: Array): ParseResults { - let mode = 'check'; + let mode: MODE = MODE.check; const parser = usage(`Usage: salty-dog [options]`) .command({ command: ['check', '*'], describe: 'validate the source documents', handler: (argi: any) => { - mode = 'check'; + mode = MODE.check; }, }) .command({ @@ -67,21 +76,21 @@ export function parseArgs(argv: Array): ParseResults { command: ['fix'], describe: 'validate the source document and insert defaults', handler: (argi: any) => { - mode = 'fix'; + mode = MODE.fix; }, }) .command({ command: ['list'], describe: 'list active rules', handler: (argi: any) => { - mode = 'list'; + mode = MODE.list; }, }) .command({ command: ['complete'], describe: 'generate tab completion script for bash or zsh', handler: (argi: any) => { - mode = 'complete'; + mode = MODE.complete; }, }) .option(CONFIG_ARGS_NAME, { @@ -110,12 +119,24 @@ export function parseArgs(argv: Array): ParseResults { default: 'yaml', type: 'string', }) - .option('rules', { - alias: ['r'], + .option('rule-file', { + alias: ['r', 'rule', 'rules'], default: [], desc: 'Rules file', type: 'array', }) + .option('rule-module', { + alias: ['m'], + default: [], + desc: 'Rules module', + type: 'array', + }) + .option('rule-path', { + alias: ['p'], + default: [], + desc: 'Rules path', + type: 'array', + }) .option('source', { alias: ['s'], default: '-', @@ -138,7 +159,9 @@ export function parseArgs(argv: Array): ParseResults { // @tslint:disable-next-line:no-any const args = parser.argv as any; - if (mode === 'complete') { + // this should not need a cast either, but something here allows TS to narrow MODE into + // MODE.check, which is much _too_ specific + if (mode as MODE === MODE.complete) { showCompletionScript(); process.exit(0); } diff --git a/src/index.ts b/src/index.ts index 32ffd64..002cb0b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,21 +1,13 @@ import { createLogger } from 'bunyan'; import { loadConfig } from './config'; -import { CONFIG_ARGS_NAME, CONFIG_ARGS_PATH, parseArgs } from './config/args'; +import { CONFIG_ARGS_NAME, CONFIG_ARGS_PATH, MODE, parseArgs, VALID_MODES } from './config/args'; import { YamlParser } from './parser/YamlParser'; -import { createRuleSelector, loadRules, resolveRules, visitRules } from './rule'; +import { createRuleSelector, createRuleSources, loadRules, resolveRules, visitRules } from './rule'; import { loadSource, writeSource } from './source'; import { VERSION_INFO } from './version'; import { VisitorContext } from './visitor/VisitorContext'; -enum MODES { - check = 'check', - fix = 'fix', - list = 'list', -} - -const MODES_LIST: Array = [MODES.check, MODES.fix, MODES.list]; - const STATUS_SUCCESS = 0; const STATUS_ERROR = 1; const STATUS_MAX = 255; @@ -29,7 +21,7 @@ export async function main(argv: Array): Promise { logger.info({ args }, 'main arguments'); // check mode - if (!MODES_LIST.includes(mode)) { + if (!VALID_MODES.has(mode)) { logger.error({ mode }, 'unsupported mode'); return STATUS_ERROR; } @@ -38,17 +30,19 @@ export async function main(argv: Array): Promise { innerOptions: { coerce: args.coerce, defaults: args.defaults, - mutate: mode === 'fix', + mutate: mode === MODE.fix, }, logger, }); - const selector = createRuleSelector(args); - const loadedRules = await loadRules(args.rules, ctx); - const activeRules = await resolveRules(loadedRules, selector); + const ruleSelector = createRuleSelector(args); + const ruleSources = createRuleSources(args); - if (mode === 'list') { - logger.info({ rules: activeRules, selector }, 'listing active rules'); + const loadedRules = await loadRules(ruleSources, ctx); + const activeRules = await resolveRules(loadedRules, ruleSelector); + + if (mode === MODE.list) { + logger.info({ activeRules, loadedRules, ruleSelector, ruleSources }, 'listing active rules'); return STATUS_SUCCESS; } diff --git a/src/rule/index.ts b/src/rule/index.ts index 8451b9a..ffc4292 100644 --- a/src/rule/index.ts +++ b/src/rule/index.ts @@ -1,9 +1,10 @@ import { applyDiff, diff } from 'deep-diff'; import { cloneDeep, Dictionary, intersection, isNil } from 'lodash'; import { LogLevel } from 'noicejs'; +import { join } from 'path'; import { YamlParser } from '../parser/YamlParser'; -import { readFile } from '../source'; +import { readDir, readFile } from '../source'; import { ensureArray, hasItems } from '../utils'; import { VisitorContext } from '../visitor/VisitorContext'; import { SchemaRule } from './SchemaRule'; @@ -22,6 +23,11 @@ export interface RuleData { select: string; } +/** + * Rule selector derived from arguments. + * + * The `excludeFoo`/`includeFoo`/ names match yargs output structure. + */ export interface RuleSelector { excludeLevel: Array; excludeName: Array; @@ -31,13 +37,30 @@ export interface RuleSelector { includeTag: Array; } -export interface RuleSource { +/** + * Rule sources derived from arguments. + * + * The `ruleFoo` names match yargs output structure. + */ +export interface RuleSources { + ruleFile: Array; + ruleModule: Array; + rulePath: Array; +} + +export interface RuleSourceData { definitions?: Dictionary; name: string; rules: Array; } -export function createRuleSelector(options: Partial) { +export interface RuleSourceModule { + definitions?: Dictionary; + name: string; + rules: Array; +} + +export function createRuleSelector(options: Partial): RuleSelector { return { excludeLevel: ensureArray(options.excludeLevel), excludeName: ensureArray(options.excludeName), @@ -48,7 +71,15 @@ export function createRuleSelector(options: Partial) { }; } -export async function loadRules(paths: Array, ctx: VisitorContext): Promise> { +export function createRuleSources(options: Partial): RuleSources { + return { + ruleFile: ensureArray(options.ruleFile), + ruleModule: ensureArray(options.ruleModule), + rulePath: ensureArray(options.rulePath), + }; +} + +export async function loadRuleFiles(paths: Array, ctx: VisitorContext): Promise> { const parser = new YamlParser(); const rules = []; @@ -57,7 +88,7 @@ export async function loadRules(paths: Array, ctx: VisitorContext): Prom encoding: 'utf-8', }); - const docs = parser.parse(contents) as Array; + const docs = parser.parse(contents) as Array; for (const data of docs) { if (!isNil(data.definitions)) { @@ -71,6 +102,52 @@ export async function loadRules(paths: Array, ctx: VisitorContext): Prom return rules; } +export async function loadRulePaths(paths: Array, ctx: VisitorContext): Promise> { + const rules = []; + + for (const path of paths) { + const allFiles = await readDir(path); + const files = allFiles.filter((name) => { + // skip files that start with `.`, limit to yml + return name.match(/^[^\.].*\.ya?ml/); + }).map((name) => join(path, name)); + + const pathRules = await loadRuleFiles(files, ctx); + rules.push(...pathRules); + } + + return rules; +} + +export async function loadRuleModules(modules: Array, ctx: VisitorContext): Promise> { + const rules = []; + + for (const name of modules) { + try { + const module: RuleSourceModule = require(name); + // TODO: ensure module has definitions, name, and rules + + if (!isNil(module.definitions)) { + ctx.addSchema(module.name, module.definitions); + } + + rules.push(...module.rules); + } catch (err) { + ctx.logger.error(err, 'error requiring rule module'); + } + } + + return rules; +} + +export async function loadRules(sources: RuleSources, ctx: VisitorContext): Promise> { + return [ + ...await loadRuleFiles(sources.ruleFile, ctx), + ...await loadRulePaths(sources.rulePath, ctx), + ...await loadRuleModules(sources.ruleModule, ctx), + ]; +} + export async function resolveRules(rules: Array, selector: RuleSelector): Promise> { const activeRules = new Set(); diff --git a/src/source.ts b/src/source.ts index 10a319b..2e47a76 100644 --- a/src/source.ts +++ b/src/source.ts @@ -1,8 +1,10 @@ -import { readFile as readBack, writeFile as writeBack } from 'fs'; +import { readdir, readFile as readBack, writeFile as writeBack } from 'fs'; import { isNil } from 'lodash'; import { promisify } from 'util'; export const FILE_ENCODING = 'utf-8'; + +export const readDir = promisify(readdir); export const readFile = promisify(readBack); export const writeFile = promisify(writeBack); diff --git a/src/utils/index.ts b/src/utils/index.ts index ac3804a..f4c0fc7 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,6 +1,6 @@ import { isNil } from 'lodash'; -export function hasItems(val: Array | null | undefined): val is Array { +export function hasItems(val: Array | null | undefined): val is Array { return (Array.isArray(val) && val.length > 0); }