From 4527249f76d7ab5da55ad5e3650773bcc27b9d23 Mon Sep 17 00:00:00 2001 From: Shigma <1700011071@pku.edu.cn> Date: Sat, 29 Sep 2018 12:17:57 +0800 Subject: [PATCH 1/7] initial commit --- packages/inline/package.json | 23 +++ packages/inline/src/index.ts | 289 ++++++++++++++++++++++++++++++++++ packages/inline/tsconfig.json | 10 ++ packages/parser/package.json | 3 +- packages/parser/src/index.ts | 1 + tsconfig.json | 1 + 6 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 packages/inline/package.json create mode 100644 packages/inline/src/index.ts create mode 100644 packages/inline/tsconfig.json diff --git a/packages/inline/package.json b/packages/inline/package.json new file mode 100644 index 0000000..ad3859e --- /dev/null +++ b/packages/inline/package.json @@ -0,0 +1,23 @@ +{ + "name": "@marklet/inline", + "version": "1.0.0", + "description": "An inline lexer for marklet.", + "author": "shigma <1700011071@pku.edu.cn>", + "contributors": [ + "jjyyxx <1449843302@qq.com>" + ], + "homepage": "https://github.com/obstudio/Marklet", + "license": "MIT", + "main": "dist/index.js", + "typings": "dist/index.d.ts", + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/obstudio/Marklet.git" + }, + "bugs": { + "url": "https://github.com/obstudio/Marklet/issues" + } +} diff --git a/packages/inline/src/index.ts b/packages/inline/src/index.ts new file mode 100644 index 0000000..c5540e9 --- /dev/null +++ b/packages/inline/src/index.ts @@ -0,0 +1,289 @@ +type ResultMap any>> = { + [key in keyof T]: ReturnType +} + +type StringLike = string | RegExp +type Capture = RegExpExecArray & ResultMap +type GetterFunction = (this: InlineLexer, capture: RegExpExecArray) => any +type GetterFunctionMap = Record +export interface LexerConfig { [key: string]: any } +export interface LexerOptions { + /** lexer capture getters */ + getters?: GetterFunctionMap + /** lexer rule regex macros */ + macros?: Record + /** entrance context */ + entrance?: string + /** default context */ + default?: string + /** assign start/end to tokens */ + requireBound?: boolean + /** other configurations */ + config?: LexerConfig +} + +export interface LexerToken { + type?: string + text?: string + content?: TokenLike[] + start?: number + end?: number + [key: string]: any +} + +export type TokenLike = string | LexerToken +interface LexerIncludeRule { include: string } +interface LexerRegexRule { + /** the regular expression to execute */ + regex?: S + /** + * a string containing all the rule flags: + * - `b`: match when the context begins + * - `e`: match end of line + * - `i`: ignore case + * - `p`: pop from the current context + * - `t`: match top level context + */ + flags?: string + /** default type of the token */ + type?: string + /** whether the rule is to be executed */ + test?: string | boolean | ((config: LexerConfig) => boolean) + /** a result token */ + token?: TokenLike | TokenLike[] | (( + this: InlineLexer, capture: Capture, content: TokenLike[], rule: this + ) => TokenLike | TokenLike[]) + /** the inner context */ + push?: string | LexerRule[] | (( + this: InlineLexer, capture: Capture + ) => string | LexerRule[] | false) + /** pop from the current context */ + pop?: boolean + /** match when the context begins */ + context_begins?: boolean + /** match top level context */ + top_level?: boolean + /** whether to ignore case */ + ignore_case?: boolean + /** match end of line */ + eol?: boolean +} + +interface LexerWarning { + message: string +} + +type LexerContext = string | NativeLexerRule[] +type LexerRule = LexerRegexRule | LexerIncludeRule +type LooseLexerRule = LexerRule +type NativeLexerRule = LexerRule +export type LexerRules = Record + +function getString(string: StringLike): string { + return string instanceof RegExp ? string.source : string +} + +export class InlineLexer { + config: LexerConfig + private rules: Record = {} + private getters: GetterFunctionMap + private entrance: string + private default: string + private requireBound: boolean + private _warnings: LexerWarning[] + private _isRunning: boolean = false + + constructor(rules: LexerRules, options: LexerOptions = {}) { + this.getters = options.getters || {} + this.config = options.config || {} + this.entrance = options.entrance || 'main' + this.default = options.default || 'text' + this.requireBound = !!options.requireBound + + const _macros = options.macros || {} + const macros: Record = {} + for (const key in _macros) { + macros[key] = getString(_macros[key]) + } + + function resolve(rule: LooseLexerRule): NativeLexerRule { + if (!('include' in rule)) { + if (rule.regex === undefined) { + rule.regex = /(?=[\s\S])/ + if (!rule.type) rule.type = 'default' + } + if (rule.test === undefined) rule.test = true + let src = getString(rule.regex) + let flags = '' + for (const key in macros) { + src = src.replace(new RegExp(`{{${key}}}`, 'g'), `(?:${macros[key]})`) + } + rule.flags = rule.flags || '' + if (rule.flags.replace(/[biept]/g, '')) { + throw new Error(`'${rule.flags}' contains invalid rule flags.`) + } + if (rule.flags.includes('p')) rule.pop = true + if (rule.flags.includes('b')) rule.context_begins = true + if (rule.flags.includes('t')) rule.top_level = true + if (rule.flags.includes('e') || rule.eol) src += ' *(?:\\n+|$)' + if (rule.flags.includes('i') || rule.ignore_case) flags += 'i' + rule.regex = new RegExp('^(?:' + src + ')', flags) + if (rule.push instanceof Array) rule.push.forEach(resolve) + } + return rule + } + + for (const key in rules) { + this.rules[key] = rules[key].map(resolve) + } + } + + private getContext(context: LexerContext): LexerRegexRule[] { + const result = typeof context === 'string' ? this.rules[context] : context + if (!result) throw new Error(`Context '${context}' was not found.`) + for (let i = result.length - 1; i >= 0; i -= 1) { + const rule: NativeLexerRule = result[i] + if ('include' in rule) { + result.splice(i, 1, ...this.getContext(rule.include)) + } + } + return []>result + } + + private _parse(source: string, context: LexerContext, isTopLevel: boolean = false): { + index: number + result: TokenLike[] + warnings: LexerWarning[] + } { + let index = 0, unmatch = '' + const result: TokenLike[] = [] + const rules = this.getContext(context) + const warnings: LexerWarning[] = this._warnings = [] + source = source.replace(/\r\n/g, '\n') + while (source) { + /** + * Matching status: + * 0. No match was found + * 1. Found match and continue + * 2. Found match and pop + */ + let status = 0 + for (const rule of rules) { + if (rule.top_level && !isTopLevel) continue + if (rule.context_begins && index) continue + + // test + let test = rule.test + if (typeof test === 'string') { + if (test.charAt(0) === '!') { + test = !this.config[test.slice(1)] + } else { + test = this.config[test] + } + } else if (typeof test === 'function') { + test = test.call(this, this.config) + } + if (!test) continue + + // regex + const capture = rule.regex.exec(source) + if (!capture) continue + source = source.slice(capture[0].length) + const start = index + index += capture[0].length + + // pop + const pop = rule.pop + status = pop ? 2 : 1 + + // push + let content: TokenLike[] = [], push = rule.push + if (typeof push === 'function') push = push.call(this, capture) + if (push) { + const subtoken = this._parse(source, push) + content = subtoken.result.map((tok) => { + if (this.requireBound && typeof tok === 'object') { + tok.start += index + tok.end += index + } + return tok + }) + warnings.concat(subtoken.warnings) + source = source.slice(subtoken.index) + index += subtoken.index + } + + // detect error + if (!pop && index === start) { + throw new Error(`Endless loop at '${ + source.slice(0, 10) + } ${ + source.length > 10 ? '...' : '' + }'.`) + } + + // resolve unmatch + if (unmatch) { + result.push(unmatch) + unmatch = '' + } + + // token + let token = rule.token + if (typeof token === 'function') { + for (const key in this.getters) { // redundant define led to some efficiency loss, consider monkey-patch RegExpExecArray or try other solutions? + Object.defineProperty(capture, key, { + get: () => this.getters[key].call(this, capture) + }) + } + token = token.call(this, capture, content) + } else if (token === undefined) { + if (push) { + token = content + } else if (!pop) { + token = capture[0] + } + } + if (token instanceof Array) token = { content: token } + if (token) { + if (typeof token === 'object') { + token.type = token.type || rule.type + if (this.requireBound) { + token.start = start + token.end = index + } + } + result.push(token) + } + + break + } + + if (!status) { + unmatch += source.charAt(0) + source = source.slice(1) + index += 1 + } + if (status === 2) break + } + + if (unmatch) result.push(unmatch) + return { index, result, warnings } + } + + pushWarning(message: string) { + this._warnings.push({ message }) + } + + parse(source: string, context?: string): TokenLike[] { + let result + if (this._isRunning) { + result = this._parse(source, context || this.default).result + } else { + this._isRunning = true + result = this._parse(source, context || this.entrance, true).result + this._isRunning = false + } + return result + } +} \ No newline at end of file diff --git a/packages/inline/tsconfig.json b/packages/inline/tsconfig.json new file mode 100644 index 0000000..e305f8d --- /dev/null +++ b/packages/inline/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "include": [ + "src" + ], + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + } +} \ No newline at end of file diff --git a/packages/parser/package.json b/packages/parser/package.json index ad9afcc..7f3c093 100644 --- a/packages/parser/package.json +++ b/packages/parser/package.json @@ -21,6 +21,7 @@ "url": "https://github.com/obstudio/Marklet/issues" }, "dependencies": { - "@marklet/core": "^1.0.4" + "@marklet/core": "^1.0.4", + "@marklet/inline": "^1.0.0" } } \ No newline at end of file diff --git a/packages/parser/src/index.ts b/packages/parser/src/index.ts index 4cee43f..c2bd63d 100644 --- a/packages/parser/src/index.ts +++ b/packages/parser/src/index.ts @@ -1,4 +1,5 @@ import { Lexer, LexerConfig, TokenLike } from '@marklet/core' +import { InlineLexer } from '@marklet/inline' function escape(html: string): string { return html diff --git a/tsconfig.json b/tsconfig.json index 6f38192..2ebebad 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,6 +11,7 @@ { "path": "./packages/core" }, { "path": "./packages/dev-server" }, { "path": "./packages/detok" }, + { "path": "./packages/inline" }, { "path": "./packages/parser" }, { "path": "./packages/syntax" } ], From 7f204f98be94f37bfb4ed5a5981e02367dc9a1b4 Mon Sep 17 00:00:00 2001 From: Shigma <1700011071@pku.edu.cn> Date: Sat, 29 Sep 2018 13:33:31 +0800 Subject: [PATCH 2/7] lexer and inline-lexer --- packages/inline/package.json | 2 +- packages/inline/src/index.ts | 267 +++++++++----------------------- packages/lexer/package.json | 26 ++++ packages/lexer/src/index.ts | 289 +++++++++++++++++++++++++++++++++++ packages/lexer/tsconfig.json | 10 ++ tsconfig.json | 1 + 6 files changed, 398 insertions(+), 197 deletions(-) create mode 100644 packages/lexer/package.json create mode 100644 packages/lexer/src/index.ts create mode 100644 packages/lexer/tsconfig.json diff --git a/packages/inline/package.json b/packages/inline/package.json index ad3859e..38935b0 100644 --- a/packages/inline/package.json +++ b/packages/inline/package.json @@ -1,7 +1,7 @@ { "name": "@marklet/inline", "version": "1.0.0", - "description": "An inline lexer for marklet.", + "description": "A fast inline lexer for marklet.", "author": "shigma <1700011071@pku.edu.cn>", "contributors": [ "jjyyxx <1449843302@qq.com>" diff --git a/packages/inline/src/index.ts b/packages/inline/src/index.ts index c5540e9..8496e55 100644 --- a/packages/inline/src/index.ts +++ b/packages/inline/src/index.ts @@ -1,39 +1,10 @@ -type ResultMap any>> = { - [key in keyof T]: ReturnType -} - type StringLike = string | RegExp -type Capture = RegExpExecArray & ResultMap -type GetterFunction = (this: InlineLexer, capture: RegExpExecArray) => any -type GetterFunctionMap = Record -export interface LexerConfig { [key: string]: any } -export interface LexerOptions { - /** lexer capture getters */ - getters?: GetterFunctionMap - /** lexer rule regex macros */ - macros?: Record - /** entrance context */ - entrance?: string - /** default context */ - default?: string - /** assign start/end to tokens */ - requireBound?: boolean - /** other configurations */ - config?: LexerConfig -} -export interface LexerToken { - type?: string - text?: string - content?: TokenLike[] - start?: number - end?: number - [key: string]: any +interface InlineCapture extends RegExpExecArray { + inner: string } -export type TokenLike = string | LexerToken -interface LexerIncludeRule { include: string } -interface LexerRegexRule { +interface InlineLexerRule { /** the regular expression to execute */ regex?: S /** @@ -42,124 +13,79 @@ interface LexerRegexRule { * - `e`: match end of line * - `i`: ignore case * - `p`: pop from the current context - * - `t`: match top level context */ flags?: string /** default type of the token */ type?: string /** whether the rule is to be executed */ - test?: string | boolean | ((config: LexerConfig) => boolean) + test?: string | boolean | ((this: InlineLexer, config: LexerConfig) => boolean) /** a result token */ - token?: TokenLike | TokenLike[] | (( - this: InlineLexer, capture: Capture, content: TokenLike[], rule: this - ) => TokenLike | TokenLike[]) - /** the inner context */ - push?: string | LexerRule[] | (( - this: InlineLexer, capture: Capture - ) => string | LexerRule[] | false) + token?: string | ((this: InlineLexer, capture: InlineCapture) => string) /** pop from the current context */ pop?: boolean /** match when the context begins */ context_begins?: boolean - /** match top level context */ - top_level?: boolean /** whether to ignore case */ ignore_case?: boolean /** match end of line */ eol?: boolean } -interface LexerWarning { - message: string -} - -type LexerContext = string | NativeLexerRule[] -type LexerRule = LexerRegexRule | LexerIncludeRule -type LooseLexerRule = LexerRule -type NativeLexerRule = LexerRule -export type LexerRules = Record - -function getString(string: StringLike): string { - return string instanceof RegExp ? string.source : string +export type LexerConfig = Record +export type InlineLexerRules = InlineLexerRule[] +export interface InlineResult { + index: number + output: string } export class InlineLexer { config: LexerConfig - private rules: Record = {} - private getters: GetterFunctionMap - private entrance: string - private default: string - private requireBound: boolean - private _warnings: LexerWarning[] - private _isRunning: boolean = false - - constructor(rules: LexerRules, options: LexerOptions = {}) { - this.getters = options.getters || {} - this.config = options.config || {} - this.entrance = options.entrance || 'main' - this.default = options.default || 'text' - this.requireBound = !!options.requireBound - - const _macros = options.macros || {} - const macros: Record = {} - for (const key in _macros) { - macros[key] = getString(_macros[key]) - } - - function resolve(rule: LooseLexerRule): NativeLexerRule { - if (!('include' in rule)) { - if (rule.regex === undefined) { - rule.regex = /(?=[\s\S])/ - if (!rule.type) rule.type = 'default' - } - if (rule.test === undefined) rule.test = true - let src = getString(rule.regex) - let flags = '' - for (const key in macros) { - src = src.replace(new RegExp(`{{${key}}}`, 'g'), `(?:${macros[key]})`) - } - rule.flags = rule.flags || '' - if (rule.flags.replace(/[biept]/g, '')) { - throw new Error(`'${rule.flags}' contains invalid rule flags.`) - } - if (rule.flags.includes('p')) rule.pop = true - if (rule.flags.includes('b')) rule.context_begins = true - if (rule.flags.includes('t')) rule.top_level = true - if (rule.flags.includes('e') || rule.eol) src += ' *(?:\\n+|$)' - if (rule.flags.includes('i') || rule.ignore_case) flags += 'i' - rule.regex = new RegExp('^(?:' + src + ')', flags) - if (rule.push instanceof Array) rule.push.forEach(resolve) + private Capture: any + private rules: InlineLexerRule[] + + constructor(rules: InlineLexerRules, config: LexerConfig = {}) { + const self = this + this.config = config + + this.Capture = class extends Array implements InlineCapture { + index: number + input: string + + constructor(array: RegExpExecArray) { + super(...array) + this.index = array.index + this.input = array.input } - return rule - } - for (const key in rules) { - this.rules[key] = rules[key].map(resolve) + get inner(): string { + const match = this.reverse().find(item => !!item) + return match ? self.parse(match).output : '' + } } - } - private getContext(context: LexerContext): LexerRegexRule[] { - const result = typeof context === 'string' ? this.rules[context] : context - if (!result) throw new Error(`Context '${context}' was not found.`) - for (let i = result.length - 1; i >= 0; i -= 1) { - const rule: NativeLexerRule = result[i] - if ('include' in rule) { - result.splice(i, 1, ...this.getContext(rule.include)) + this.rules = rules.map((rule) => { + if (rule.regex === undefined) { + rule.regex = /(?=[\s\S])/ + if (!rule.type) rule.type = 'default' } - } - return []>result + if (rule.test === undefined) rule.test = true + let src = rule.regex instanceof RegExp ? rule.regex.source : rule.regex + let flags = '' + rule.flags = rule.flags || '' + if (rule.flags.replace(/[biep]/g, '')) { + throw new Error(`'${rule.flags}' contains invalid rule flags.`) + } + if (rule.flags.includes('p')) rule.pop = true + if (rule.flags.includes('b')) rule.context_begins = true + if (rule.flags.includes('e') || rule.eol) src += ' *(?:\\n+|$)' + if (rule.flags.includes('i') || rule.ignore_case) flags += 'i' + rule.regex = new RegExp('^(?:' + src + ')', flags) + return rule as InlineLexerRule + }) } - private _parse(source: string, context: LexerContext, isTopLevel: boolean = false): { - index: number - result: TokenLike[] - warnings: LexerWarning[] - } { - let index = 0, unmatch = '' - const result: TokenLike[] = [] - const rules = this.getContext(context) - const warnings: LexerWarning[] = this._warnings = [] - source = source.replace(/\r\n/g, '\n') + private _parse(source: string): InlineResult { + let index = 0, unmatch = '', output = '' while (source) { /** * Matching status: @@ -168,8 +94,7 @@ export class InlineLexer { * 2. Found match and pop */ let status = 0 - for (const rule of rules) { - if (rule.top_level && !isTopLevel) continue + for (const rule of this.rules) { if (rule.context_begins && index) continue // test @@ -178,7 +103,7 @@ export class InlineLexer { if (test.charAt(0) === '!') { test = !this.config[test.slice(1)] } else { - test = this.config[test] + test = !!this.config[test] } } else if (typeof test === 'function') { test = test.call(this, this.config) @@ -186,104 +111,54 @@ export class InlineLexer { if (!test) continue // regex - const capture = rule.regex.exec(source) - if (!capture) continue + const match = rule.regex.exec(source) + if (!match) continue + if (!match[0].length) { + throw new Error(`Endless loop at '${ + source.slice(0, 10) + } ${ + source.length > 10 ? '...' : '' + }'.`) + } + const capture = new this.Capture(match) source = source.slice(capture[0].length) - const start = index index += capture[0].length // pop const pop = rule.pop status = pop ? 2 : 1 - // push - let content: TokenLike[] = [], push = rule.push - if (typeof push === 'function') push = push.call(this, capture) - if (push) { - const subtoken = this._parse(source, push) - content = subtoken.result.map((tok) => { - if (this.requireBound && typeof tok === 'object') { - tok.start += index - tok.end += index - } - return tok - }) - warnings.concat(subtoken.warnings) - source = source.slice(subtoken.index) - index += subtoken.index - } - - // detect error - if (!pop && index === start) { - throw new Error(`Endless loop at '${ - source.slice(0, 10) - } ${ - source.length > 10 ? '...' : '' - }'.`) - } - // resolve unmatch if (unmatch) { - result.push(unmatch) + output += unmatch unmatch = '' } // token let token = rule.token if (typeof token === 'function') { - for (const key in this.getters) { // redundant define led to some efficiency loss, consider monkey-patch RegExpExecArray or try other solutions? - Object.defineProperty(capture, key, { - get: () => this.getters[key].call(this, capture) - }) - } - token = token.call(this, capture, content) + token = token.call(this, capture) } else if (token === undefined) { - if (push) { - token = content - } else if (!pop) { - token = capture[0] - } - } - if (token instanceof Array) token = { content: token } - if (token) { - if (typeof token === 'object') { - token.type = token.type || rule.type - if (this.requireBound) { - token.start = start - token.end = index - } - } - result.push(token) + token = capture[0] } + output += token break } - if (!status) { + if (status === 2) break + if (status === 0) { unmatch += source.charAt(0) source = source.slice(1) index += 1 } - if (status === 2) break } - if (unmatch) result.push(unmatch) - return { index, result, warnings } - } - - pushWarning(message: string) { - this._warnings.push({ message }) + if (unmatch) output += unmatch + return { index, output } } - parse(source: string, context?: string): TokenLike[] { - let result - if (this._isRunning) { - result = this._parse(source, context || this.default).result - } else { - this._isRunning = true - result = this._parse(source, context || this.entrance, true).result - this._isRunning = false - } - return result + parse(source: string): InlineResult { + return this._parse(source.replace(/\r\n/g, '\n')) } } \ No newline at end of file diff --git a/packages/lexer/package.json b/packages/lexer/package.json new file mode 100644 index 0000000..863b5d0 --- /dev/null +++ b/packages/lexer/package.json @@ -0,0 +1,26 @@ +{ + "name": "@marklet/lexer", + "version": "1.0.9", + "description": "A document lexer for marklet.", + "author": "shigma <1700011071@pku.edu.cn>", + "contributors": [ + "jjyyxx <1449843302@qq.com>" + ], + "homepage": "https://github.com/obstudio/Marklet", + "license": "MIT", + "main": "dist/index.js", + "typings": "dist/index.d.ts", + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/obstudio/Marklet.git" + }, + "bugs": { + "url": "https://github.com/obstudio/Marklet/issues" + }, + "dependencies": { + "@marklet/core": "^1.0.9" + } +} \ No newline at end of file diff --git a/packages/lexer/src/index.ts b/packages/lexer/src/index.ts new file mode 100644 index 0000000..6aad324 --- /dev/null +++ b/packages/lexer/src/index.ts @@ -0,0 +1,289 @@ +type StringLike = string | RegExp +function getString(string: StringLike): string { + return string instanceof RegExp ? string.source : string +} + +type ResultMap any>> = { + [key in keyof T]: ReturnType +} + +type Capture = RegExpExecArray & ResultMap +type GetterFunction = (this: Lexer, capture: RegExpExecArray) => any +type GetterFunctionMap = Record +export interface LexerConfig { [key: string]: any } +export interface LexerOptions { + /** lexer capture getters */ + getters?: GetterFunctionMap + /** lexer rule regex macros */ + macros?: Record + /** entrance context */ + entrance?: string + /** default context */ + default?: string + /** assign start/end to tokens */ + requireBound?: boolean + /** other configurations */ + config?: LexerConfig +} + +export interface LexerToken { + type?: string + text?: string + content?: TokenLike[] + start?: number + end?: number + [key: string]: any +} + +export type TokenLike = string | LexerToken +interface LexerIncludeRule { include: string } +interface LexerRegexRule { + /** the regular expression to execute */ + regex?: S + /** + * a string containing all the rule flags: + * - `b`: match when the context begins + * - `e`: match end of line + * - `i`: ignore case + * - `p`: pop from the current context + * - `t`: match top level context + */ + flags?: string + /** default type of the token */ + type?: string + /** whether the rule is to be executed */ + test?: string | boolean | ((this: Lexer, config: LexerConfig) => boolean) + /** a result token */ + token?: TokenLike | TokenLike[] | (( + this: Lexer, capture: Capture, content: TokenLike[] + ) => TokenLike | TokenLike[]) + /** the inner context */ + push?: string | LexerRule[] | (( + this: Lexer, capture: Capture + ) => string | LexerRule[] | false) + /** pop from the current context */ + pop?: boolean + /** match when the context begins */ + context_begins?: boolean + /** match top level context */ + top_level?: boolean + /** whether to ignore case */ + ignore_case?: boolean + /** match end of line */ + eol?: boolean +} + +interface LexerWarning { + message: string +} + +type LexerContext = string | NativeLexerRule[] +type LexerRule = LexerRegexRule | LexerIncludeRule +type LooseLexerRule = LexerRule +type NativeLexerRule = LexerRule +export type LexerRules = Record + +export class Lexer { + config: LexerConfig + private rules: Record = {} + private getters: GetterFunctionMap + private entrance: string + private default: string + private requireBound: boolean + private _warnings: LexerWarning[] + private _isRunning: boolean = false + + constructor(rules: LexerRules, options: LexerOptions = {}) { + this.getters = options.getters || {} + this.config = options.config || {} + this.entrance = options.entrance || 'main' + this.default = options.default || 'text' + this.requireBound = !!options.requireBound + + const _macros = options.macros || {} + const macros: Record = {} + for (const key in _macros) { + macros[key] = getString(_macros[key]) + } + + function resolve(rule: LooseLexerRule): NativeLexerRule { + if (!('include' in rule)) { + if (rule.regex === undefined) { + rule.regex = /(?=[\s\S])/ + if (!rule.type) rule.type = 'default' + } + if (rule.test === undefined) rule.test = true + let src = getString(rule.regex) + let flags = '' + for (const key in macros) { + src = src.replace(new RegExp(`{{${key}}}`, 'g'), `(?:${macros[key]})`) + } + rule.flags = rule.flags || '' + if (rule.flags.replace(/[biept]/g, '')) { + throw new Error(`'${rule.flags}' contains invalid rule flags.`) + } + if (rule.flags.includes('p')) rule.pop = true + if (rule.flags.includes('b')) rule.context_begins = true + if (rule.flags.includes('t')) rule.top_level = true + if (rule.flags.includes('e') || rule.eol) src += ' *(?:\\n+|$)' + if (rule.flags.includes('i') || rule.ignore_case) flags += 'i' + rule.regex = new RegExp('^(?:' + src + ')', flags) + if (rule.push instanceof Array) rule.push.forEach(resolve) + } + return rule as NativeLexerRule + } + + for (const key in rules) { + this.rules[key] = rules[key].map(resolve) + } + } + + private getContext(context: LexerContext): LexerRegexRule[] { + const result = typeof context === 'string' ? this.rules[context] : context + if (!result) throw new Error(`Context '${context}' was not found.`) + for (let i = result.length - 1; i >= 0; i -= 1) { + const rule: NativeLexerRule = result[i] + if ('include' in rule) { + result.splice(i, 1, ...this.getContext(rule.include)) + } + } + return []>result + } + + private _parse(source: string, context: LexerContext, isTopLevel: boolean = false): { + index: number + result: TokenLike[] + warnings: LexerWarning[] + } { + let index = 0, unmatch = '' + const result: TokenLike[] = [] + const rules = this.getContext(context) + const warnings: LexerWarning[] = this._warnings = [] + while (source) { + /** + * Matching status: + * 0. No match was found + * 1. Found match and continue + * 2. Found match and pop + */ + let status = 0 + for (const rule of rules) { + if (rule.top_level && !isTopLevel) continue + if (rule.context_begins && index) continue + + // test + let test = rule.test + if (typeof test === 'string') { + if (test.charAt(0) === '!') { + test = !this.config[test.slice(1)] + } else { + test = !!this.config[test] + } + } else if (typeof test === 'function') { + test = test.call(this, this.config) + } + if (!test) continue + + // regex + const capture = rule.regex.exec(source) + if (!capture) continue + source = source.slice(capture[0].length) + const start = index + index += capture[0].length + + // pop + const pop = rule.pop + status = pop ? 2 : 1 + + // push + let content: TokenLike[] = [], push = rule.push + if (typeof push === 'function') push = push.call(this, capture) + if (push) { + const subtoken = this._parse(source, push) + content = subtoken.result.map((tok) => { + if (this.requireBound && typeof tok === 'object') { + tok.start += index + tok.end += index + } + return tok + }) + warnings.concat(subtoken.warnings) + source = source.slice(subtoken.index) + index += subtoken.index + } + + // detect error + if (!pop && index === start) { + throw new Error(`Endless loop at '${ + source.slice(0, 10) + } ${ + source.length > 10 ? '...' : '' + }'.`) + } + + // resolve unmatch + if (unmatch) { + result.push(unmatch) + unmatch = '' + } + + // token + let token = rule.token + if (typeof token === 'function') { + for (const key in this.getters) { // redundant define led to some efficiency loss, consider monkey-patch RegExpExecArray or try other solutions? + Object.defineProperty(capture, key, { + get: () => this.getters[key].call(this, capture) + }) + } + token = token.call(this, capture, content) + } else if (token === undefined) { + if (push) { + token = content + } else if (!pop) { + token = capture[0] + } + } + if (token instanceof Array) token = { content: token } + if (token) { + if (typeof token === 'object') { + token.type = token.type || rule.type + if (this.requireBound) { + token.start = start + token.end = index + } + } + result.push(token) + } + + break + } + + if (status === 2) break + if (status === 0) { + unmatch += source.charAt(0) + source = source.slice(1) + index += 1 + } + } + + if (unmatch) result.push(unmatch) + return { index, result, warnings } + } + + pushWarning(message: string) { + this._warnings.push({ message }) + } + + parse(source: string, context?: string): TokenLike[] { + let result + source = source.replace(/\r\n/g, '\n') + if (this._isRunning) { + result = this._parse(source, context || this.default).result + } else { + this._isRunning = true + result = this._parse(source, context || this.entrance, true).result + this._isRunning = false + } + return result + } +} \ No newline at end of file diff --git a/packages/lexer/tsconfig.json b/packages/lexer/tsconfig.json new file mode 100644 index 0000000..e305f8d --- /dev/null +++ b/packages/lexer/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "include": [ + "src" + ], + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + } +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 2ebebad..6cd3d2f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,6 +12,7 @@ { "path": "./packages/dev-server" }, { "path": "./packages/detok" }, { "path": "./packages/inline" }, + { "path": "./packages/lexer" }, { "path": "./packages/parser" }, { "path": "./packages/syntax" } ], From 34f6254c0c1066c4ac0120ac647ae3ec4acaa44e Mon Sep 17 00:00:00 2001 From: Shigma <1700011071@pku.edu.cn> Date: Sat, 29 Sep 2018 15:29:58 +0800 Subject: [PATCH 3/7] abstraction --- packages/core/package.json | 4 +- packages/core/src/index.ts | 294 +++++++---------------------------- packages/inline/package.json | 3 + packages/inline/src/index.ts | 123 +++++---------- packages/lexer/package.json | 2 +- packages/lexer/src/index.ts | 156 ++++--------------- 6 files changed, 132 insertions(+), 450 deletions(-) diff --git a/packages/core/package.json b/packages/core/package.json index 6a5da86..55f4ab4 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@marklet/core", - "version": "1.0.9", - "description": "A lexer for marklet.", + "version": "2.0.0-beta.0", + "description": "Some core conceptions of marklet.", "author": "shigma <1700011071@pku.edu.cn>", "contributors": [ "jjyyxx <1449843302@qq.com>" diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 1c481e1..3ec3a3b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,27 +1,9 @@ -type ResultMap any>> = { - [key in keyof T]: ReturnType -} +export type StringLike = string | RegExp -type StringLike = string | RegExp -type Capture = RegExpExecArray & ResultMap -type GetterFunction = (this: Lexer, capture: RegExpExecArray) => any -type GetterFunctionMap = Record -export interface LexerConfig { [key: string]: any } -export interface LexerOptions { - /** lexer capture getters */ - getters?: GetterFunctionMap - /** lexer rule regex macros */ - macros?: Record - /** entrance context */ - entrance?: string - /** default context */ - default?: string - /** assign start/end to tokens */ - requireBound?: boolean - /** other configurations */ - config?: LexerConfig -} +export type LexerConfig = Record +export type LexerMacros = Record +export type TokenLike = string | LexerToken export interface LexerToken { type?: string text?: string @@ -31,9 +13,18 @@ export interface LexerToken { [key: string]: any } -export type TokenLike = string | LexerToken -interface LexerIncludeRule { include: string } -interface LexerRegexRule { +export type LexerRule< + S extends StringLike = RegExp, + T extends LexerClass = LexerClass, + R extends RegExpExecArray = RegExpExecArray, +> = LexerIncludeRule | LexerRegexRule + +export interface LexerIncludeRule { include: string } +export interface LexerRegexRule< + S extends StringLike = StringLike, + T extends LexerClass = LexerClass, + R extends RegExpExecArray = RegExpExecArray, +> { /** the regular expression to execute */ regex?: S /** @@ -48,15 +39,13 @@ interface LexerRegexRule { /** default type of the token */ type?: string /** whether the rule is to be executed */ - test?: string | boolean | ((config: LexerConfig) => boolean) + test?: string | boolean | ((this: T, config: LexerConfig) => boolean) /** a result token */ token?: TokenLike | TokenLike[] | (( - this: Lexer, capture: Capture, content: TokenLike[], rule: this + this: T, capture: R, content: TokenLike[] ) => TokenLike | TokenLike[]) /** the inner context */ - push?: string | LexerRule[] | (( - this: Lexer, capture: Capture - ) => string | LexerRule[] | false) + push?: string | LexerRule[] /** pop from the current context */ pop?: boolean /** match when the context begins */ @@ -69,221 +58,48 @@ interface LexerRegexRule { eol?: boolean } -interface LexerWarning { - message: string -} - -type LexerContext = string | NativeLexerRule[] -type LexerRule = LexerRegexRule | LexerIncludeRule -type LooseLexerRule = LexerRule -type NativeLexerRule = LexerRule -export type LexerRules = Record - -function getString(string: StringLike): string { +/** Transform a string-like object into a raw string. */ +export function getString(string: StringLike): string { return string instanceof RegExp ? string.source : string } -export class Lexer { - config: LexerConfig - private rules: Record = {} - private getters: GetterFunctionMap - private entrance: string - private default: string - private requireBound: boolean - private _warnings: LexerWarning[] - private _isRunning: boolean = false - - constructor(rules: LexerRules, options: LexerOptions = {}) { - this.getters = options.getters || {} - this.config = options.config || {} - this.entrance = options.entrance || 'main' - this.default = options.default || 'text' - this.requireBound = !!options.requireBound - - const _macros = options.macros || {} - const macros: Record = {} - for (const key in _macros) { - macros[key] = getString(_macros[key]) +export function parseRule(rule: LexerRule, macros: LexerMacros = {}): LexerRule { + if (!('include' in rule)) { + if (rule.regex === undefined) { + rule.regex = /(?=[\s\S])/ + if (!rule.type) rule.type = 'default' } - - function resolve(rule: LooseLexerRule): NativeLexerRule { - if (!('include' in rule)) { - if (rule.regex === undefined) { - rule.regex = /(?=[\s\S])/ - if (!rule.type) rule.type = 'default' - } - if (rule.test === undefined) rule.test = true - let src = getString(rule.regex) - let flags = '' - for (const key in macros) { - src = src.replace(new RegExp(`{{${key}}}`, 'g'), `(?:${macros[key]})`) - } - rule.flags = rule.flags || '' - if (rule.flags.replace(/[biept]/g, '')) { - throw new Error(`'${rule.flags}' contains invalid rule flags.`) - } - if (rule.flags.includes('p')) rule.pop = true - if (rule.flags.includes('b')) rule.context_begins = true - if (rule.flags.includes('t')) rule.top_level = true - if (rule.flags.includes('e') || rule.eol) src += ' *(?:\\n+|$)' - if (rule.flags.includes('i') || rule.ignore_case) flags += 'i' - rule.regex = new RegExp('^(?:' + src + ')', flags) - if (rule.push instanceof Array) rule.push.forEach(resolve) - } - return rule + if (rule.test === undefined) rule.test = true + let src = getString(rule.regex) + let flags = '' + for (const key in macros) { + src = src.replace(new RegExp(`{{${key}}}`, 'g'), `(?:${macros[key]})`) } - - for (const key in rules) { - this.rules[key] = rules[key].map(resolve) + rule.flags = rule.flags || '' + if (rule.flags.replace(/[biept]/g, '')) { + throw new Error(`'${rule.flags}' contains invalid rule flags.`) } + if (rule.flags.includes('p')) rule.pop = true + if (rule.flags.includes('b')) rule.context_begins = true + if (rule.flags.includes('t')) rule.top_level = true + if (rule.flags.includes('e') || rule.eol) src += ' *(?:\\n+|$)' + if (rule.flags.includes('i') || rule.ignore_case) flags += 'i' + rule.regex = new RegExp('^(?:' + src + ')', flags) + if (rule.push instanceof Array) rule.push.forEach(_rule => parseRule(_rule, macros)) } + return rule as LexerRule +} - private getContext(context: LexerContext): LexerRegexRule[] { - const result = typeof context === 'string' ? this.rules[context] : context - if (!result) throw new Error(`Context '${context}' was not found.`) - for (let i = result.length - 1; i >= 0; i -= 1) { - const rule: NativeLexerRule = result[i] - if ('include' in rule) { - result.splice(i, 1, ...this.getContext(rule.include)) - } - } - return []>result - } - - private _parse(source: string, context: LexerContext, isTopLevel: boolean = false): { - index: number - result: TokenLike[] - warnings: LexerWarning[] - } { - let index = 0, unmatch = '' - const result: TokenLike[] = [] - const rules = this.getContext(context) - const warnings: LexerWarning[] = this._warnings = [] - source = source.replace(/\r\n/g, '\n') - while (source) { - /** - * Matching status: - * 0. No match was found - * 1. Found match and continue - * 2. Found match and pop - */ - let status = 0 - for (const rule of rules) { - if (rule.top_level && !isTopLevel) continue - if (rule.context_begins && index) continue - - // test - let test = rule.test - if (typeof test === 'string') { - if (test.charAt(0) === '!') { - test = !this.config[test.slice(1)] - } else { - test = this.config[test] - } - } else if (typeof test === 'function') { - test = test.call(this, this.config) - } - if (!test) continue - - // regex - const capture = rule.regex.exec(source) - if (!capture) continue - source = source.slice(capture[0].length) - const start = index - index += capture[0].length - - // pop - const pop = rule.pop - status = pop ? 2 : 1 - - // push - let content: TokenLike[] = [], push = rule.push - if (typeof push === 'function') push = push.call(this, capture) - if (push) { - const subtoken = this._parse(source, push) - content = subtoken.result.map((tok) => { - if (this.requireBound && typeof tok === 'object') { - tok.start += index - tok.end += index - } - return tok - }) - warnings.concat(subtoken.warnings) - source = source.slice(subtoken.index) - index += subtoken.index - } - - // detect error - if (!pop && index === start) { - throw new Error(`Endless loop at '${ - source.slice(0, 10) - } ${ - source.length > 10 ? '...' : '' - }'.`) - } - - // resolve unmatch - if (unmatch) { - result.push(unmatch) - unmatch = '' - } - - // token - let token = rule.token - if (typeof token === 'function') { - for (const key in this.getters) { // redundant define led to some efficiency loss, consider monkey-patch RegExpExecArray or try other solutions? - Object.defineProperty(capture, key, { - get: () => this.getters[key].call(this, capture) - }) - } - token = token.call(this, capture, content) - } else if (token === undefined) { - if (push) { - token = content - } else if (!pop) { - token = capture[0] - } - } - if (token instanceof Array) token = { content: token } - if (token) { - if (typeof token === 'object') { - token.type = token.type || rule.type - if (this.requireBound) { - token.start = start - token.end = index - } - } - result.push(token) - } - - break - } - - if (!status) { - unmatch += source.charAt(0) - source = source.slice(1) - index += 1 - } - if (status === 2) break - } - - if (unmatch) result.push(unmatch) - return { index, result, warnings } - } - - pushWarning(message: string) { - this._warnings.push({ message }) - } +export interface LexerClass { + config: LexerConfig + parse(source: string): any +} - parse(source: string, context?: string): TokenLike[] { - let result - if (this._isRunning) { - result = this._parse(source, context || this.default).result - } else { - this._isRunning = true - result = this._parse(source, context || this.entrance, true).result - this._isRunning = false - } - return result - } -} \ No newline at end of file +export enum MatchStatus { + /** No match was found */ + NO_MATCH, + /** Found match and continue */ + CONTINUE, + /** Found match and pop */ + POP, +} diff --git a/packages/inline/package.json b/packages/inline/package.json index 38935b0..0bef1b0 100644 --- a/packages/inline/package.json +++ b/packages/inline/package.json @@ -19,5 +19,8 @@ }, "bugs": { "url": "https://github.com/obstudio/Marklet/issues" + }, + "dependencies": { + "@marklet/core": "^2.0.0-beta.0" } } diff --git a/packages/inline/src/index.ts b/packages/inline/src/index.ts index 8496e55..38e8fd5 100644 --- a/packages/inline/src/index.ts +++ b/packages/inline/src/index.ts @@ -1,99 +1,53 @@ -type StringLike = string | RegExp +import { + StringLike, + LexerConfig, + LexerClass, + LexerRegexRule, + MatchStatus, + parseRule, +} from '@marklet/core' + +export { LexerConfig } + +class InlineCapture extends Array implements RegExpExecArray { + index: number + input: string + lexer: InlineLexer + + constructor(lexer: InlineLexer, array: RegExpExecArray) { + super(...array) + this.lexer = lexer + this.index = array.index + this.input = array.input + } -interface InlineCapture extends RegExpExecArray { - inner: string + get inner(): string { + const match = this.reverse().find(item => !!item) + return match ? this.lexer.parse(match).output : '' + } } -interface InlineLexerRule { - /** the regular expression to execute */ - regex?: S - /** - * a string containing all the rule flags: - * - `b`: match when the context begins - * - `e`: match end of line - * - `i`: ignore case - * - `p`: pop from the current context - */ - flags?: string - /** default type of the token */ - type?: string - /** whether the rule is to be executed */ - test?: string | boolean | ((this: InlineLexer, config: LexerConfig) => boolean) - /** a result token */ - token?: string | ((this: InlineLexer, capture: InlineCapture) => string) - /** pop from the current context */ - pop?: boolean - /** match when the context begins */ - context_begins?: boolean - /** whether to ignore case */ - ignore_case?: boolean - /** match end of line */ - eol?: boolean -} +type InlineLexerRule = LexerRegexRule -export type LexerConfig = Record export type InlineLexerRules = InlineLexerRule[] -export interface InlineResult { +export interface InlineLexerResult { index: number output: string } -export class InlineLexer { +export class InlineLexer implements LexerClass { config: LexerConfig - private Capture: any - private rules: InlineLexerRule[] + private rules: InlineLexerRule[] constructor(rules: InlineLexerRules, config: LexerConfig = {}) { - const self = this + this.rules = rules.map(rule => parseRule(rule) as InlineLexerRule) this.config = config - - this.Capture = class extends Array implements InlineCapture { - index: number - input: string - - constructor(array: RegExpExecArray) { - super(...array) - this.index = array.index - this.input = array.input - } - - get inner(): string { - const match = this.reverse().find(item => !!item) - return match ? self.parse(match).output : '' - } - } - - this.rules = rules.map((rule) => { - if (rule.regex === undefined) { - rule.regex = /(?=[\s\S])/ - if (!rule.type) rule.type = 'default' - } - if (rule.test === undefined) rule.test = true - let src = rule.regex instanceof RegExp ? rule.regex.source : rule.regex - let flags = '' - rule.flags = rule.flags || '' - if (rule.flags.replace(/[biep]/g, '')) { - throw new Error(`'${rule.flags}' contains invalid rule flags.`) - } - if (rule.flags.includes('p')) rule.pop = true - if (rule.flags.includes('b')) rule.context_begins = true - if (rule.flags.includes('e') || rule.eol) src += ' *(?:\\n+|$)' - if (rule.flags.includes('i') || rule.ignore_case) flags += 'i' - rule.regex = new RegExp('^(?:' + src + ')', flags) - return rule as InlineLexerRule - }) } - private _parse(source: string): InlineResult { + private _parse(source: string): InlineLexerResult { let index = 0, unmatch = '', output = '' while (source) { - /** - * Matching status: - * 0. No match was found - * 1. Found match and continue - * 2. Found match and pop - */ - let status = 0 + let status: MatchStatus = MatchStatus.NO_MATCH for (const rule of this.rules) { if (rule.context_begins && index) continue @@ -120,13 +74,12 @@ export class InlineLexer { source.length > 10 ? '...' : '' }'.`) } - const capture = new this.Capture(match) + const capture = new InlineCapture(this, match) source = source.slice(capture[0].length) index += capture[0].length // pop - const pop = rule.pop - status = pop ? 2 : 1 + status = rule.pop ? MatchStatus.POP : MatchStatus.CONTINUE // resolve unmatch if (unmatch) { @@ -146,8 +99,8 @@ export class InlineLexer { break } - if (status === 2) break - if (status === 0) { + if (status === MatchStatus.POP) break + if (status === MatchStatus.NO_MATCH) { unmatch += source.charAt(0) source = source.slice(1) index += 1 @@ -158,7 +111,7 @@ export class InlineLexer { return { index, output } } - parse(source: string): InlineResult { + parse(source: string): InlineLexerResult { return this._parse(source.replace(/\r\n/g, '\n')) } } \ No newline at end of file diff --git a/packages/lexer/package.json b/packages/lexer/package.json index 863b5d0..8c49a9f 100644 --- a/packages/lexer/package.json +++ b/packages/lexer/package.json @@ -21,6 +21,6 @@ "url": "https://github.com/obstudio/Marklet/issues" }, "dependencies": { - "@marklet/core": "^1.0.9" + "@marklet/core": "^2.0.0-beta" } } \ No newline at end of file diff --git a/packages/lexer/src/index.ts b/packages/lexer/src/index.ts index 6aad324..995c283 100644 --- a/packages/lexer/src/index.ts +++ b/packages/lexer/src/index.ts @@ -1,21 +1,21 @@ -type StringLike = string | RegExp -function getString(string: StringLike): string { - return string instanceof RegExp ? string.source : string -} - -type ResultMap any>> = { - [key in keyof T]: ReturnType -} +import { + StringLike, + LexerMacros, + LexerConfig, + LexerClass, + LexerRule, + TokenLike, + LexerRegexRule, + MatchStatus, + parseRule, + getString, +} from '@marklet/core' + +export { LexerConfig } -type Capture = RegExpExecArray & ResultMap -type GetterFunction = (this: Lexer, capture: RegExpExecArray) => any -type GetterFunctionMap = Record -export interface LexerConfig { [key: string]: any } export interface LexerOptions { - /** lexer capture getters */ - getters?: GetterFunctionMap /** lexer rule regex macros */ - macros?: Record + macros?: LexerMacros /** entrance context */ entrance?: string /** default context */ @@ -26,67 +26,22 @@ export interface LexerOptions { config?: LexerConfig } -export interface LexerToken { - type?: string - text?: string - content?: TokenLike[] - start?: number - end?: number - [key: string]: any -} - -export type TokenLike = string | LexerToken -interface LexerIncludeRule { include: string } -interface LexerRegexRule { - /** the regular expression to execute */ - regex?: S - /** - * a string containing all the rule flags: - * - `b`: match when the context begins - * - `e`: match end of line - * - `i`: ignore case - * - `p`: pop from the current context - * - `t`: match top level context - */ - flags?: string - /** default type of the token */ - type?: string - /** whether the rule is to be executed */ - test?: string | boolean | ((this: Lexer, config: LexerConfig) => boolean) - /** a result token */ - token?: TokenLike | TokenLike[] | (( - this: Lexer, capture: Capture, content: TokenLike[] - ) => TokenLike | TokenLike[]) - /** the inner context */ - push?: string | LexerRule[] | (( - this: Lexer, capture: Capture - ) => string | LexerRule[] | false) - /** pop from the current context */ - pop?: boolean - /** match when the context begins */ - context_begins?: boolean - /** match top level context */ - top_level?: boolean - /** whether to ignore case */ - ignore_case?: boolean - /** match end of line */ - eol?: boolean -} - interface LexerWarning { message: string } -type LexerContext = string | NativeLexerRule[] -type LexerRule = LexerRegexRule | LexerIncludeRule -type LooseLexerRule = LexerRule -type NativeLexerRule = LexerRule -export type LexerRules = Record +type LexerContext = string | LexerRule[] +export type LexerRules = Record[]> + +interface LexerResult { + index: number + result: TokenLike[] + warnings: LexerWarning[] +} -export class Lexer { +export class Lexer implements LexerClass { config: LexerConfig - private rules: Record = {} - private getters: GetterFunctionMap + private rules: Record = {} private entrance: string private default: string private requireBound: boolean @@ -94,7 +49,6 @@ export class Lexer { private _isRunning: boolean = false constructor(rules: LexerRules, options: LexerOptions = {}) { - this.getters = options.getters || {} this.config = options.config || {} this.entrance = options.entrance || 'main' this.default = options.default || 'text' @@ -105,36 +59,8 @@ export class Lexer { for (const key in _macros) { macros[key] = getString(_macros[key]) } - - function resolve(rule: LooseLexerRule): NativeLexerRule { - if (!('include' in rule)) { - if (rule.regex === undefined) { - rule.regex = /(?=[\s\S])/ - if (!rule.type) rule.type = 'default' - } - if (rule.test === undefined) rule.test = true - let src = getString(rule.regex) - let flags = '' - for (const key in macros) { - src = src.replace(new RegExp(`{{${key}}}`, 'g'), `(?:${macros[key]})`) - } - rule.flags = rule.flags || '' - if (rule.flags.replace(/[biept]/g, '')) { - throw new Error(`'${rule.flags}' contains invalid rule flags.`) - } - if (rule.flags.includes('p')) rule.pop = true - if (rule.flags.includes('b')) rule.context_begins = true - if (rule.flags.includes('t')) rule.top_level = true - if (rule.flags.includes('e') || rule.eol) src += ' *(?:\\n+|$)' - if (rule.flags.includes('i') || rule.ignore_case) flags += 'i' - rule.regex = new RegExp('^(?:' + src + ')', flags) - if (rule.push instanceof Array) rule.push.forEach(resolve) - } - return rule as NativeLexerRule - } - for (const key in rules) { - this.rules[key] = rules[key].map(resolve) + this.rules[key] = rules[key].map(rule => parseRule(rule, macros)) } } @@ -142,31 +68,21 @@ export class Lexer { const result = typeof context === 'string' ? this.rules[context] : context if (!result) throw new Error(`Context '${context}' was not found.`) for (let i = result.length - 1; i >= 0; i -= 1) { - const rule: NativeLexerRule = result[i] + const rule: LexerRule = result[i] if ('include' in rule) { result.splice(i, 1, ...this.getContext(rule.include)) } } - return []>result + return result as LexerRegexRule[] } - private _parse(source: string, context: LexerContext, isTopLevel: boolean = false): { - index: number - result: TokenLike[] - warnings: LexerWarning[] - } { + private _parse(source: string, context: LexerContext, isTopLevel: boolean = false): LexerResult { let index = 0, unmatch = '' const result: TokenLike[] = [] const rules = this.getContext(context) const warnings: LexerWarning[] = this._warnings = [] while (source) { - /** - * Matching status: - * 0. No match was found - * 1. Found match and continue - * 2. Found match and pop - */ - let status = 0 + let status: MatchStatus = MatchStatus.NO_MATCH for (const rule of rules) { if (rule.top_level && !isTopLevel) continue if (rule.context_begins && index) continue @@ -193,11 +109,10 @@ export class Lexer { // pop const pop = rule.pop - status = pop ? 2 : 1 + status = pop ? MatchStatus.POP : MatchStatus.CONTINUE // push let content: TokenLike[] = [], push = rule.push - if (typeof push === 'function') push = push.call(this, capture) if (push) { const subtoken = this._parse(source, push) content = subtoken.result.map((tok) => { @@ -230,11 +145,6 @@ export class Lexer { // token let token = rule.token if (typeof token === 'function') { - for (const key in this.getters) { // redundant define led to some efficiency loss, consider monkey-patch RegExpExecArray or try other solutions? - Object.defineProperty(capture, key, { - get: () => this.getters[key].call(this, capture) - }) - } token = token.call(this, capture, content) } else if (token === undefined) { if (push) { @@ -258,8 +168,8 @@ export class Lexer { break } - if (status === 2) break - if (status === 0) { + if (status === MatchStatus.POP) break + if (status === MatchStatus.NO_MATCH) { unmatch += source.charAt(0) source = source.slice(1) index += 1 From 0ba6a297ea997f77e26931b55497cb2bf65b25e2 Mon Sep 17 00:00:00 2001 From: Shigma <1700011071@pku.edu.cn> Date: Sat, 29 Sep 2018 18:23:22 +0800 Subject: [PATCH 4/7] almost finished --- packages/core/src/index.ts | 25 ++- packages/inline/src/index.ts | 15 +- packages/lexer/src/index.ts | 115 ++++++----- packages/parser/package.json | 3 +- packages/parser/src/index.ts | 366 ++++++++++++++++++--------------- packages/renderer/package.json | 2 +- 6 files changed, 297 insertions(+), 229 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 3ec3a3b..9f71c60 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -15,14 +15,14 @@ export interface LexerToken { export type LexerRule< S extends StringLike = RegExp, - T extends LexerClass = LexerClass, + T extends LexerInstance = LexerInstance, R extends RegExpExecArray = RegExpExecArray, > = LexerIncludeRule | LexerRegexRule export interface LexerIncludeRule { include: string } export interface LexerRegexRule< - S extends StringLike = StringLike, - T extends LexerClass = LexerClass, + S extends StringLike = RegExp, + T extends LexerInstance = LexerInstance, R extends RegExpExecArray = RegExpExecArray, > { /** the regular expression to execute */ @@ -33,6 +33,7 @@ export interface LexerRegexRule< * - `e`: match end of line * - `i`: ignore case * - `p`: pop from the current context + * - `s`: pop when no match is found * - `t`: match top level context */ flags?: string @@ -45,9 +46,11 @@ export interface LexerRegexRule< this: T, capture: R, content: TokenLike[] ) => TokenLike | TokenLike[]) /** the inner context */ - push?: string | LexerRule[] + push?: string | LexerRule[] /** pop from the current context */ pop?: boolean + /** pop when no match is found */ + strict?: boolean /** match when the context begins */ context_begins?: boolean /** match top level context */ @@ -76,10 +79,11 @@ export function parseRule(rule: LexerRule, macros: LexerMacros = {}) src = src.replace(new RegExp(`{{${key}}}`, 'g'), `(?:${macros[key]})`) } rule.flags = rule.flags || '' - if (rule.flags.replace(/[biept]/g, '')) { + if (rule.flags.replace(/[biepst]/g, '')) { throw new Error(`'${rule.flags}' contains invalid rule flags.`) } if (rule.flags.includes('p')) rule.pop = true + if (rule.flags.includes('s')) rule.strict = true if (rule.flags.includes('b')) rule.context_begins = true if (rule.flags.includes('t')) rule.top_level = true if (rule.flags.includes('e') || rule.eol) src += ' *(?:\\n+|$)' @@ -90,11 +94,20 @@ export function parseRule(rule: LexerRule, macros: LexerMacros = {}) return rule as LexerRule } -export interface LexerClass { +export interface LexerInstance { config: LexerConfig parse(source: string): any } +export interface InlineLexerResult { + index: number + output: string +} + +export interface InlineLexerInstance extends LexerInstance { + parse(source: string): InlineLexerResult +} + export enum MatchStatus { /** No match was found */ NO_MATCH, diff --git a/packages/inline/src/index.ts b/packages/inline/src/index.ts index 38e8fd5..b52b68d 100644 --- a/packages/inline/src/index.ts +++ b/packages/inline/src/index.ts @@ -1,14 +1,13 @@ import { StringLike, LexerConfig, - LexerClass, LexerRegexRule, + InlineLexerInstance, + InlineLexerResult, MatchStatus, parseRule, } from '@marklet/core' -export { LexerConfig } - class InlineCapture extends Array implements RegExpExecArray { index: number input: string @@ -30,18 +29,14 @@ class InlineCapture extends Array implements RegExpExecArray { type InlineLexerRule = LexerRegexRule export type InlineLexerRules = InlineLexerRule[] -export interface InlineLexerResult { - index: number - output: string -} -export class InlineLexer implements LexerClass { +export class InlineLexer implements InlineLexerInstance { config: LexerConfig private rules: InlineLexerRule[] constructor(rules: InlineLexerRules, config: LexerConfig = {}) { this.rules = rules.map(rule => parseRule(rule) as InlineLexerRule) - this.config = config + this.config = config || {} } private _parse(source: string): InlineLexerResult { @@ -67,7 +62,7 @@ export class InlineLexer implements LexerClass { // regex const match = rule.regex.exec(source) if (!match) continue - if (!match[0].length) { + if (!match[0].length && !rule.pop) { throw new Error(`Endless loop at '${ source.slice(0, 10) } ${ diff --git a/packages/lexer/src/index.ts b/packages/lexer/src/index.ts index 995c283..27faa37 100644 --- a/packages/lexer/src/index.ts +++ b/packages/lexer/src/index.ts @@ -2,56 +2,48 @@ import { StringLike, LexerMacros, LexerConfig, - LexerClass, LexerRule, - TokenLike, + LexerInstance, LexerRegexRule, + InlineLexerInstance, + TokenLike, MatchStatus, parseRule, getString, } from '@marklet/core' -export { LexerConfig } - export interface LexerOptions { /** lexer rule regex macros */ macros?: LexerMacros /** entrance context */ entrance?: string - /** default context */ - default?: string + /** default inline context */ + inlineEntrance?: string /** assign start/end to tokens */ requireBound?: boolean /** other configurations */ config?: LexerConfig } -interface LexerWarning { - message: string -} - -type LexerContext = string | LexerRule[] -export type LexerRules = Record[]> +type NativeLexerContext = LexerRegexRule[] | InlineLexerInstance +export type LexerContexts = Record[] | InlineLexerInstance> interface LexerResult { index: number result: TokenLike[] - warnings: LexerWarning[] } -export class Lexer implements LexerClass { +export class Lexer implements LexerInstance { config: LexerConfig - private rules: Record = {} + private contexts: Record = {} private entrance: string - private default: string + private inlineEntrance: string private requireBound: boolean - private _warnings: LexerWarning[] - private _isRunning: boolean = false - constructor(rules: LexerRules, options: LexerOptions = {}) { + constructor(contexts: LexerContexts, options: LexerOptions = {}) { this.config = options.config || {} this.entrance = options.entrance || 'main' - this.default = options.default || 'text' + this.inlineEntrance = options.inlineEntrance || 'text' this.requireBound = !!options.requireBound const _macros = options.macros || {} @@ -59,31 +51,59 @@ export class Lexer implements LexerClass { for (const key in _macros) { macros[key] = getString(_macros[key]) } - for (const key in rules) { - this.rules[key] = rules[key].map(rule => parseRule(rule, macros)) + for (const key in contexts) { + const context = contexts[key] + this.contexts[key] = context instanceof Array + ? context.map(rule => parseRule(rule, macros)) + : context } } - private getContext(context: LexerContext): LexerRegexRule[] { - const result = typeof context === 'string' ? this.rules[context] : context + private getContext(context: string | InlineLexerInstance | LexerRule[], strictMode?: boolean) { + const result = typeof context === 'string' ? this.contexts[context] : context if (!result) throw new Error(`Context '${context}' was not found.`) - for (let i = result.length - 1; i >= 0; i -= 1) { - const rule: LexerRule = result[i] - if ('include' in rule) { - result.splice(i, 1, ...this.getContext(rule.include)) + if (result instanceof Array) { + for (let i = result.length - 1; i >= 0; i -= 1) { + const rule: LexerRule = result[i] + if ('include' in rule) { + const includes = this.getContext(rule.include) + if (includes instanceof Array) { + result.splice(i, 1, ...includes) + } else { + result.splice(i, 1, { + regex: /^(?=[\s\S])/, + push: rule.include, + strict: true, + }) + } + } + } + if (strictMode) { + result.push({ + regex: /^(?=[\s\S])/, + pop: true, + }) } } - return result as LexerRegexRule[] + return result as NativeLexerContext } - private _parse(source: string, context: LexerContext, isTopLevel: boolean = false): LexerResult { + private _parse(source: string, context: NativeLexerContext, isTopLevel?: boolean): LexerResult { let index = 0, unmatch = '' const result: TokenLike[] = [] - const rules = this.getContext(context) - const warnings: LexerWarning[] = this._warnings = [] + + // apply inline lexer + if (!(context instanceof Array)) { + const result = context.parse(source) + return { + index: result.index, + result: [result.output], + } + } + while (source) { let status: MatchStatus = MatchStatus.NO_MATCH - for (const rule of rules) { + for (const rule of context) { if (rule.top_level && !isTopLevel) continue if (rule.context_begins && index) continue @@ -114,7 +134,8 @@ export class Lexer implements LexerClass { // push let content: TokenLike[] = [], push = rule.push if (push) { - const subtoken = this._parse(source, push) + const context = this.getContext(push, rule.strict) + const subtoken = this._parse(source, context) content = subtoken.result.map((tok) => { if (this.requireBound && typeof tok === 'object') { tok.start += index @@ -122,7 +143,6 @@ export class Lexer implements LexerClass { } return tok }) - warnings.concat(subtoken.warnings) source = source.slice(subtoken.index) index += subtoken.index } @@ -177,23 +197,20 @@ export class Lexer implements LexerClass { } if (unmatch) result.push(unmatch) - return { index, result, warnings } + return { index, result } } - pushWarning(message: string) { - this._warnings.push({ message }) + inline(source: string, context: string = this.inlineEntrance): string { + const inlineContext = this.getContext(context) + if (inlineContext instanceof Array) { + throw new Error(`'${context}' is not a inline context.`) + } + return inlineContext.parse(source).output } - parse(source: string, context?: string): TokenLike[] { - let result + parse(source: string, context: string = this.entrance): TokenLike[] { + const initialContext = this.getContext(context) source = source.replace(/\r\n/g, '\n') - if (this._isRunning) { - result = this._parse(source, context || this.default).result - } else { - this._isRunning = true - result = this._parse(source, context || this.entrance, true).result - this._isRunning = false - } - return result + return this._parse(source, initialContext, true).result } -} \ No newline at end of file +} diff --git a/packages/parser/package.json b/packages/parser/package.json index 7f3c093..9df4d42 100644 --- a/packages/parser/package.json +++ b/packages/parser/package.json @@ -21,7 +21,8 @@ "url": "https://github.com/obstudio/Marklet/issues" }, "dependencies": { - "@marklet/core": "^1.0.4", + "@marklet/core": "^2.0.0-beta.0", + "@marklet/lexer": "^1.0.9", "@marklet/inline": "^1.0.0" } } \ No newline at end of file diff --git a/packages/parser/src/index.ts b/packages/parser/src/index.ts index c2bd63d..798dc7a 100644 --- a/packages/parser/src/index.ts +++ b/packages/parser/src/index.ts @@ -1,5 +1,6 @@ -import { Lexer, LexerConfig, TokenLike } from '@marklet/core' +import { LexerConfig, TokenLike, InlineLexerInstance } from '@marklet/core' import { InlineLexer } from '@marklet/inline' +import { Lexer } from '@marklet/lexer' function escape(html: string): string { return html @@ -23,183 +24,59 @@ interface MarkletLexerConfig extends LexerConfig { default_language?: string } -class MarkletLexer extends Lexer { +class MarkletInlineLexer extends InlineLexer { constructor(config: MarkletLexerConfig = {}) { - super({ - main: [{ - type: 'newline', - regex: /\n+/, - token: null - }, { - type: 'heading', - regex: /(#{1,4}) +([^\n]+?)( +#)?/, - eol: true, - token(cap) { - let text, center - if (this.config.header_align && cap[3]) { - text = this.parse(cap[2], 'text').join('') - center = true - } else { - text = this.parse(cap[2] + (cap[3] || ''), 'text').join('') - center = false - } - return { level: cap[1].length, text, center } - } - }, { - type: 'section', - test: 'allow_section', - regex: /(\^{1,4}) +([^\n]+?)/, - eol: true, - push: 'main', - token(cap) { - const text = this.parse(cap[2], 'text').join('') - return { level: cap[1].length, text } - } - }, { - type: 'quote', - regex: />([\w-]*) +/, - push: 'block', - token: (cap, content) => ({ style: cap[1], content }) - }, { - type: 'separator', - regex: / *([-=])(\1|\.\1| \1)\2+/, - eol: true, - token: (cap) => ({ - thick: cap[1] === '=', - style: cap[2].length === 1 ? 'normal' - : cap[2][0] === ' ' ? 'dashed' : 'dotted' - }) - }, { - type: 'codeblock', - regex: / *(`{3,}) *([\w-]+)? *\n([\s\S]*?)\n? *\1/, - eol: true, - token(cap) { - return { - lang: cap[2] || this.config.default_language, - text: cap[3] || '', - } - } - }, { - type: 'usages', - regex: /(?= *\? +\S)/, - push: [{ - type: 'usage', - regex: / *\? +([^\n]+?)/, - eol: true, - push: [{ - regex: /(?= *\? )/, - pop: true - }, { - include: 'text' - }], - token(cap, cont) { - return { - text: this.parse(cap[1], 'text').join(''), - content: cont - } - } - }, { - pop: true - }] - }, { - type: 'list', - regex: / *(?={{bullet}} +[^\n]+)/, - push: [{ - type: 'item', - regex: /( *)({{bullet}}) +(?=[^\n]+)/, - push: [{ - regex: /\n? *(?={{bullet}} +[^\n]+)/, - pop: true - }, { - include: 'text' - }], - token(cap, cont) { - return { - text: cont.join(''), - ordered: cap[2].length > 1, - indent: cap[1].length, - } - } - }, { - pop: true - }], - token: (_, cont) => collect(cont) - }, { - type: 'inlinelist', - regex: /(?=\+)/, - push: [{ - type: 'item', - regex: /\+/, - push: [{ - regex: /\+?$|\+\n(?=\+)|\+?(?=\n)|(?=\+)/, - pop: true - }, { - include: 'text' - }], - token(_, cont) { - return cont.join('') - } - }, { - regex: /\n|$/, - pop: true - }], - token: (_, cont) => ({ content: cont }) - }, { - type: 'table', - regex: /$^/, // FIXME: placeholder for syntax discussion - push: [], - token: (_, cont) => ({ content: cont }) - }, { - type: 'paragraph', - push: 'text', - token: (_, cont) => ({ text: cont.join('') }) - }], - block: [{ - regex: /\n[ \t]*\n/, + super([ + { + regex: /(?=\n[ \t]*(\n|$))/, pop: true - }, { - include: 'main' - }], - text: [{ + }, + { type: 'escape', regex: /\\([\s\S])/, token: (cap) => cap[1] - }, { - regex: /(?=\n[ \t]*(\n|$))/, - pop: true - }, { + }, + { type: 'newline', regex: /\n/, token: '
' - }, { + }, + { type: 'code', regex: /(`+)\s*([\s\S]*?[^`]?)\s*\1(?!`)/, token: (cap) => `${escape(cap[2])}` - }, { + }, + { type: 'strikeout', regex: /-(?=\S)([\s\S]*?\S)-(?!-)/, - token: (cap) => `${cap.next}` - }, { + token: (cap) => `${cap.inner}` + }, + { type: 'underline', regex: /_(?=\S)([\s\S]*?\S)_(?!_)/, - token: (cap) => `${cap.next}` - }, { + token: (cap) => `${cap.inner}` + }, + { type: 'bold', regex: /\*\*(?=\S)([\s\S]*?\S)\*\*(?!\*)/, - token: (cap) => `${cap.next}` - }, { + token: (cap) => `${cap.inner}` + }, + { type: 'italic', regex: /\*(?=\S)([\s\S]*?\S)\*(?!\*)/, - token: (cap) => `${cap.next}` - }, { + token: (cap) => `${cap.inner}` + }, + { type: 'comment', regex: /\(\((?=\S)([\s\S]*?\S)\)\)(?!\))/, - token: (cap) => `${cap.next}` - }, { + token: (cap) => `${cap.inner}` + }, + { type: 'package', regex: /{{(?=\S)([\s\S]*?\S)}}(?!})/, - token: (cap) => `${cap.next}` - }, { + token: (cap) => `${cap.inner}` + }, + { type: 'link', regex: /\[(?:([^\]|]+)\|)?([^\]]+)\]/, token(cap) { @@ -219,17 +96,181 @@ class MarkletLexer extends Lexer { `${text}` : // TODO: special treatment like necessary? `${text}` } - }] + } + ], config) + } +} + +class MarkletLexer extends Lexer { + constructor(config: MarkletLexerConfig = {}) { + super({ + text: new MarkletInlineLexer(config), + main: [ + { + type: 'newline', + regex: /\n+/, + token: null + }, + { + type: 'heading', + regex: /(#{1,4}) +([^\n]+?)( +#)?/, + eol: true, + token(cap) { + let text, center + if (this.config.header_align && cap[3]) { + text = this.inline(cap[2]) + center = true + } else { + text = this.inline(cap[2] + (cap[3] || '')) + center = false + } + return { level: cap[1].length, text, center } + } + }, + { + type: 'section', + test: 'allow_section', + regex: /(\^{1,4}) +([^\n]+?)/, + eol: true, + push: 'main', + token(cap) { + return { + level: cap[1].length, + text: this.inline(cap[2]), + } + } + }, + { + type: 'quote', + regex: />([\w-]*) +/, + push: 'block', + token: (cap, content) => ({ style: cap[1], content }) + }, + { + type: 'separator', + regex: / *([-=])(\1|\.\1| \1)\2+/, + eol: true, + token: (cap) => ({ + thick: cap[1] === '=', + style: cap[2].length === 1 ? 'normal' + : cap[2][0] === ' ' ? 'dashed' : 'dotted' + }) + }, + { + type: 'codeblock', + regex: / *(`{3,}) *([\w-]+)? *\n([\s\S]*?)\n? *\1/, + eol: true, + token(cap) { + return { + lang: cap[2] || this.config.default_language, + text: cap[3] || '', + } + } + }, + { + type: 'usages', + regex: /(?= *\? +\S)/, + strict: true, + push: [ + { + type: 'usage', + regex: / *\? +([^\n]+?)/, + eol: true, + push: [ + { + regex: /(?= *\? )/, + pop: true + }, + { + include: 'text' + } + ], + token(cap, cont) { + return { + text: this.inline(cap[1]), + content: cont, + } + } + } + ] + }, + { + type: 'list', + regex: / *(?={{bullet}} +[^\n]+)/, + strict: true, + push: [ + { + type: 'item', + regex: /( *)({{bullet}}) +(?=[^\n]+)/, + push: [{ + regex: /\n? *(?={{bullet}} +[^\n]+)/, + pop: true + }, { + include: 'text' + }], + token(cap, cont) { + return { + text: cont.join(''), + ordered: cap[2].length > 1, + indent: cap[1].length, + } + } + } + ], + token: (_, cont) => collect(cont) + }, + { + type: 'inlinelist', + regex: /(?=\+)/, + push: [ + { + type: 'item', + regex: /\+/, + push: [ + { + regex: /\+?$|\+\n(?=\+)|\+?(?=\n)|(?=\+)/, + pop: true + }, + { + include: 'text' + } + ], + token(_, cont) { + return cont.join('') + } + }, + { + regex: /\n|$/, + pop: true + } + ], + token: (_, cont) => ({ content: cont }) + }, + { + type: 'table', + regex: /$^/, // FIXME: placeholder for syntax discussion + push: [], + token: (_, cont) => ({ content: cont }) + }, + { + type: 'paragraph', + push: 'text', + token: (_, cont) => ({ text: cont.join('') }) + } + ], + block: [ + { + regex: /\n[ \t]*\n/, + pop: true + }, + { + include: 'main' + } + ], }, { macros: { bullet: /-|\d+\./, }, - getters: { - next(capture) { - const result = this.parse(capture.reverse().find(item => !!item) || '') - return result/* .map(token => token.text || token) */.join('') - }, - }, config: { header_align: true, allow_section: true, @@ -257,5 +298,6 @@ export function parse(options: ParseOptions): TokenLike[] { export { MarkletLexer as Lexer, + MarkletInlineLexer as InlineLexer, MarkletLexerConfig as LexerConfig, } diff --git a/packages/renderer/package.json b/packages/renderer/package.json index e8928a7..15538f3 100644 --- a/packages/renderer/package.json +++ b/packages/renderer/package.json @@ -21,7 +21,7 @@ "url": "https://github.com/obstudio/Marklet/issues" }, "dependencies": { - "@marklet/core": "^1.0.9" + "@marklet/core": "^2.0.0-beta.0" }, "peerDependencies": { "vue": "^2.5.17" From 00e5542e9f2da389feae9760555739edf9cc14af Mon Sep 17 00:00:00 2001 From: Shigma <1700011071@pku.edu.cn> Date: Sat, 29 Sep 2018 18:31:27 +0800 Subject: [PATCH 5/7] base fix --- packages/core/tsconfig.json | 2 +- packages/detok/tsconfig.json | 2 +- packages/dev-server/tsconfig.json | 2 +- packages/inline/tsconfig.json | 2 +- packages/lexer/tsconfig.json | 2 +- packages/parser/tsconfig.json | 2 +- packages/syntax/tsconfig.json | 2 +- tsconfig.base.json | 10 ++++++++++ tsconfig.json | 8 -------- 9 files changed, 17 insertions(+), 15 deletions(-) create mode 100644 tsconfig.base.json diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index e305f8d..8faa3c3 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.json", + "extends": "../../tsconfig.base.json", "include": [ "src" ], diff --git a/packages/detok/tsconfig.json b/packages/detok/tsconfig.json index 4e48185..9677f08 100644 --- a/packages/detok/tsconfig.json +++ b/packages/detok/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.json", + "extends": "../../tsconfig.base.json", "include": [ "src" ], diff --git a/packages/dev-server/tsconfig.json b/packages/dev-server/tsconfig.json index 59a996a..30874db 100644 --- a/packages/dev-server/tsconfig.json +++ b/packages/dev-server/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.json", + "extends": "../../tsconfig.base.json", "include": [ "src" ], diff --git a/packages/inline/tsconfig.json b/packages/inline/tsconfig.json index e305f8d..8faa3c3 100644 --- a/packages/inline/tsconfig.json +++ b/packages/inline/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.json", + "extends": "../../tsconfig.base.json", "include": [ "src" ], diff --git a/packages/lexer/tsconfig.json b/packages/lexer/tsconfig.json index e305f8d..8faa3c3 100644 --- a/packages/lexer/tsconfig.json +++ b/packages/lexer/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.json", + "extends": "../../tsconfig.base.json", "include": [ "src" ], diff --git a/packages/parser/tsconfig.json b/packages/parser/tsconfig.json index 4e48185..9677f08 100644 --- a/packages/parser/tsconfig.json +++ b/packages/parser/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.json", + "extends": "../../tsconfig.base.json", "include": [ "src" ], diff --git a/packages/syntax/tsconfig.json b/packages/syntax/tsconfig.json index 4e48185..9677f08 100644 --- a/packages/syntax/tsconfig.json +++ b/packages/syntax/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.json", + "extends": "../../tsconfig.base.json", "include": [ "src" ], diff --git a/tsconfig.base.json b/tsconfig.base.json new file mode 100644 index 0000000..77c864d --- /dev/null +++ b/tsconfig.base.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "module": "commonjs", + "declaration": true, + "target": "es2018", + "noImplicitAny": true, + "composite": true, + "esModuleInterop": true + } +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 6cd3d2f..6dc9dbf 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,12 +1,4 @@ { - "compilerOptions": { - "module": "commonjs", - "declaration": true, - "target": "es2018", - "noImplicitAny": true, - "composite": true, - "esModuleInterop": true - }, "references": [ { "path": "./packages/core" }, { "path": "./packages/dev-server" }, From c390293cc1c1f43a09e61a39d33d77e55f80288d Mon Sep 17 00:00:00 2001 From: Shigma <1700011071@pku.edu.cn> Date: Sat, 29 Sep 2018 19:44:53 +0800 Subject: [PATCH 6/7] bump version --- build/publish.js | 52 +++++++++++++++++--------------- packages/cli/package.json | 6 ++-- packages/core/package.json | 2 +- packages/detok/package.json | 4 +-- packages/dev-server/package.json | 4 +-- packages/inline/package.json | 2 +- packages/lexer/package.json | 4 +-- packages/marklet/package.json | 6 ++-- packages/parser/package.json | 6 ++-- packages/parser/src/index.ts | 2 +- packages/syntax/package.json | 4 +-- packages/syntax/src/index.ts | 4 +-- packages/test/package.json | 4 +-- 13 files changed, 52 insertions(+), 48 deletions(-) diff --git a/build/publish.js b/build/publish.js index 42b6224..caa4999 100644 --- a/build/publish.js +++ b/build/publish.js @@ -4,7 +4,7 @@ const chalk = require('chalk') const semver = require('semver') const cp = require('child_process') const program = require('commander') -const { exec, execSync } = require('./util') +const { exec } = require('./util') function toVersion(version) { return `${version.major}.${version.minor}.${version.patch}` @@ -13,12 +13,11 @@ function toVersion(version) { class Package { constructor(name) { this.current = require(`../packages/${name}/package.json`) - this.previous = JSON.parse(cp.execSync(`git show HEAD:packages/${name}/package.json`).toString('utf8')) + this.previous = JSON.parse(cp.execSync(`git show HEAD:packages/${name}/package.json`).toString().trim()) this.major = semver.major(this.previous.version) this.minor = semver.minor(this.previous.version) this.patch = semver.patch(this.previous.version) this.newVersion = this.current.version - delete this.current.gitHead } bump(flag) { @@ -48,10 +47,10 @@ packageNames.forEach(name => packages[name] = new Package(name)) program .usage('[options] [names...]') .option('-a, --all') - .option('-M, --major') - .option('-m, --minor') - .option('-p, --patch') - .option('-N, --no-npm') + .option('-1, --major') + .option('-2, --minor') + .option('-3, --patch') + .option('-p, --publish') .parse(process.argv) const flag = program.major ? 'major' : program.minor ? 'minor' : 'patch' @@ -83,25 +82,30 @@ ${chalk.cyanBright(packages[name].newVersion)}`) let counter = 0, promise = Promise.resolve(), failed = false -if (program.npm) { - packageNames.forEach((name) => { - if (packages[name].newVersion !== packages[name].previous.version) { - if (packages[name].current.private) return - const npmVersion = execSync(`npm show ${packages[name].current.name} version`) - if (semver.gte(npmVersion, packages[name].newVersion)) return - counter += 1 - fs.writeFileSync( - path.join(__dirname, `../packages/${name}/package.json`), - JSON.stringify(packages[name], null, 2), - ) - promise = promise.then((code) => { - failed = failed || code - return exec(`cd packages/${name} && npm publish`) - }) - } - }) +if (program.publish) { + console.log('\nWaiting for packages to publish ...') } +packageNames.forEach((name) => { + if (packages[name].newVersion !== packages[name].previous.version) { + fs.writeFileSync( + path.join(__dirname, `../packages/${name}/package.json`), + JSON.stringify(packages[name], null, 2), + ) + if (packages[name].current.private || !program.publish) return + const npmVersion = cp.execSync(`npm show ${packages[name].current.name} version`).toString().trim() + if (semver.gte(npmVersion, packages[name].newVersion)) return + console.log(` - ${name} (${packages[name].current.name}): \ +${chalk.green(npmVersion)} => \ +${chalk.greenBright(packages[name].newVersion)}`) + counter += 1 + promise = promise.then((code) => { + failed = failed || code + return exec(`cd packages/${name} && npm publish`) + }) + } +}) + promise.then(() => { if (!counter) { console.log('No packages to publish.') diff --git a/packages/cli/package.json b/packages/cli/package.json index 29f6b57..0acb1ba 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@marklet/cli", - "version": "1.1.4", + "version": "1.1.5", "description": "A command line interface for marklet.", "author": "jjyyxx <1449843302@qq.com>", "contributors": [ @@ -19,8 +19,8 @@ "url": "https://github.com/obstudio/Marklet/issues" }, "dependencies": { - "@marklet/dev-server": "^1.0.11", - "@marklet/parser": "^1.0.4", + "@marklet/dev-server": "^1.0.12", + "@marklet/parser": "^1.1.0", "chalk": "^2.4.1", "commander": "^2.18.0", "js-yaml": "^3.12.0" diff --git a/packages/core/package.json b/packages/core/package.json index 55f4ab4..c294c16 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@marklet/core", - "version": "2.0.0-beta.0", + "version": "2.0.0", "description": "Some core conceptions of marklet.", "author": "shigma <1700011071@pku.edu.cn>", "contributors": [ diff --git a/packages/detok/package.json b/packages/detok/package.json index 4284baa..dfdb3af 100644 --- a/packages/detok/package.json +++ b/packages/detok/package.json @@ -1,6 +1,6 @@ { "name": "@marklet/detok", - "version": "1.0.9", + "version": "1.0.10", "description": "A detokenizer for marklet.", "author": "jjyyxx <1449843302@qq.com>", "contributors": [ @@ -24,6 +24,6 @@ "cheerio": "^1.0.0-rc.2" }, "devDependencies": { - "@marklet/core": "^1.0.4" + "@marklet/core": "^2.0.0" } } \ No newline at end of file diff --git a/packages/dev-server/package.json b/packages/dev-server/package.json index 23a25ae..4b2c846 100644 --- a/packages/dev-server/package.json +++ b/packages/dev-server/package.json @@ -1,6 +1,6 @@ { "name": "@marklet/dev-server", - "version": "1.0.11", + "version": "1.0.12", "description": "A develop server for marklet.", "author": "jjyyxx <1449843302@qq.com>", "contributors": [ @@ -21,7 +21,7 @@ "url": "https://github.com/obstudio/Marklet/issues" }, "dependencies": { - "@marklet/parser": "^1.0.4", + "@marklet/parser": "^1.1.0", "@marklet/renderer": "^1.1.2", "vue": "^2.5.17", "ws": "^6.0.0" diff --git a/packages/inline/package.json b/packages/inline/package.json index 0bef1b0..e6038e5 100644 --- a/packages/inline/package.json +++ b/packages/inline/package.json @@ -21,6 +21,6 @@ "url": "https://github.com/obstudio/Marklet/issues" }, "dependencies": { - "@marklet/core": "^2.0.0-beta.0" + "@marklet/core": "^2.0.0" } } diff --git a/packages/lexer/package.json b/packages/lexer/package.json index 8c49a9f..5cf1771 100644 --- a/packages/lexer/package.json +++ b/packages/lexer/package.json @@ -1,6 +1,6 @@ { "name": "@marklet/lexer", - "version": "1.0.9", + "version": "1.0.10", "description": "A document lexer for marklet.", "author": "shigma <1700011071@pku.edu.cn>", "contributors": [ @@ -21,6 +21,6 @@ "url": "https://github.com/obstudio/Marklet/issues" }, "dependencies": { - "@marklet/core": "^2.0.0-beta" + "@marklet/core": "^2.0.0" } } \ No newline at end of file diff --git a/packages/marklet/package.json b/packages/marklet/package.json index 2a64db9..e9262d3 100644 --- a/packages/marklet/package.json +++ b/packages/marklet/package.json @@ -1,6 +1,6 @@ { "name": "markletjs", - "version": "1.1.12", + "version": "1.1.13", "description": "A markup language designed for API manual pages.", "author": "jjyyxx <1449843302@qq.com>", "contributors": [ @@ -28,8 +28,8 @@ "url": "https://github.com/obstudio/Marklet/issues" }, "dependencies": { - "@marklet/cli": "^1.1.4", - "@marklet/parser": "^1.0.4", + "@marklet/cli": "^1.1.5", + "@marklet/parser": "^1.1.0", "@marklet/renderer": "^1.1.2" } } \ No newline at end of file diff --git a/packages/parser/package.json b/packages/parser/package.json index 9df4d42..a6e8865 100644 --- a/packages/parser/package.json +++ b/packages/parser/package.json @@ -1,6 +1,6 @@ { "name": "@marklet/parser", - "version": "1.0.9", + "version": "1.1.0", "description": "A document lexer for marklet.", "author": "shigma <1700011071@pku.edu.cn>", "contributors": [ @@ -21,8 +21,8 @@ "url": "https://github.com/obstudio/Marklet/issues" }, "dependencies": { - "@marklet/core": "^2.0.0-beta.0", - "@marklet/lexer": "^1.0.9", + "@marklet/core": "^2.0.0", + "@marklet/lexer": "^1.0.10", "@marklet/inline": "^1.0.0" } } \ No newline at end of file diff --git a/packages/parser/src/index.ts b/packages/parser/src/index.ts index 798dc7a..488a4ef 100644 --- a/packages/parser/src/index.ts +++ b/packages/parser/src/index.ts @@ -1,4 +1,4 @@ -import { LexerConfig, TokenLike, InlineLexerInstance } from '@marklet/core' +import { LexerConfig, TokenLike } from '@marklet/core' import { InlineLexer } from '@marklet/inline' import { Lexer } from '@marklet/lexer' diff --git a/packages/syntax/package.json b/packages/syntax/package.json index d676b55..9000d29 100644 --- a/packages/syntax/package.json +++ b/packages/syntax/package.json @@ -1,6 +1,6 @@ { "name": "@marklet/syntax", - "version": "1.0.9", + "version": "1.0.10", "description": "A common language lexer for marklet.", "author": "shigma <1700011071@pku.edu.cn>", "homepage": "https://github.com/obstudio/Marklet", @@ -18,7 +18,7 @@ "url": "https://github.com/obstudio/Marklet/issues" }, "dependencies": { - "@marklet/core": "^1.0.4", + "@marklet/lexer": "^1.0.10", "js-yaml": "^3.12.0" } } \ No newline at end of file diff --git a/packages/syntax/src/index.ts b/packages/syntax/src/index.ts index 5993151..7ee1b73 100644 --- a/packages/syntax/src/index.ts +++ b/packages/syntax/src/index.ts @@ -1,4 +1,4 @@ -import { Lexer, LexerRules } from '@marklet/core' +import { Lexer, LexerContexts } from '@marklet/lexer' type SyntaxRule = SyntaxMetaRule | SyntaxIncludeRule | SyntaxRegexRule interface SyntaxToken { scope: string, text: string } @@ -48,6 +48,6 @@ export class SyntaxLexer extends Lexer { }) } for (const key in contexts) traverse(contexts[key]) - super(contexts, { macros }) + super(contexts as LexerContexts, { macros }) } } diff --git a/packages/test/package.json b/packages/test/package.json index ec964a9..8a6ecc3 100644 --- a/packages/test/package.json +++ b/packages/test/package.json @@ -1,6 +1,6 @@ { "name": "@marklet/test", - "version": "1.0.9", + "version": "1.0.10", "private": true, "author": "jjyyxx <1449843302@qq.com>", "homepage": "https://github.com/obstudio/Marklet", @@ -17,6 +17,6 @@ }, "dependencies": { "@marklet/detok": "^1.0.4", - "markletjs": "^1.1.4" + "markletjs": "^1.1.13" } } \ No newline at end of file From a0ebf201b6dc8d2d063a0123ee90b20d5addc07d Mon Sep 17 00:00:00 2001 From: jjyyxx Date: Sat, 29 Sep 2018 20:38:57 +0800 Subject: [PATCH 7/7] fix incorrect build order --- .vscode/settings.json | 2 +- packages/inline/tsconfig.json | 5 ++++- packages/lexer/tsconfig.json | 5 ++++- packages/parser/tsconfig.json | 4 +++- packages/syntax/tsconfig.json | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 4c96c16..42b0849 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,7 +8,7 @@ "**/package-lock.json": true, "package.json.lerna_backup": true, "packages/**/node_modules": true, - "packages/**/tsconfig.json": true + // "packages/**/tsconfig.json": true }, "editor.wordSeparators": "`~!@#%^&*()-=+[{]}\\|;:'\",.<>/?", } \ No newline at end of file diff --git a/packages/inline/tsconfig.json b/packages/inline/tsconfig.json index 8faa3c3..9677f08 100644 --- a/packages/inline/tsconfig.json +++ b/packages/inline/tsconfig.json @@ -6,5 +6,8 @@ "compilerOptions": { "outDir": "dist", "rootDir": "src" - } + }, + "references": [ + { "path": "../core" } + ] } \ No newline at end of file diff --git a/packages/lexer/tsconfig.json b/packages/lexer/tsconfig.json index 8faa3c3..9677f08 100644 --- a/packages/lexer/tsconfig.json +++ b/packages/lexer/tsconfig.json @@ -6,5 +6,8 @@ "compilerOptions": { "outDir": "dist", "rootDir": "src" - } + }, + "references": [ + { "path": "../core" } + ] } \ No newline at end of file diff --git a/packages/parser/tsconfig.json b/packages/parser/tsconfig.json index 9677f08..8a214b6 100644 --- a/packages/parser/tsconfig.json +++ b/packages/parser/tsconfig.json @@ -8,6 +8,8 @@ "rootDir": "src" }, "references": [ - { "path": "../core" } + { "path": "../core" }, + { "path": "../inline" }, + { "path": "../lexer" } ] } \ No newline at end of file diff --git a/packages/syntax/tsconfig.json b/packages/syntax/tsconfig.json index 9677f08..004deda 100644 --- a/packages/syntax/tsconfig.json +++ b/packages/syntax/tsconfig.json @@ -8,6 +8,6 @@ "rootDir": "src" }, "references": [ - { "path": "../core" } + { "path": "../lexer" } ] } \ No newline at end of file