diff --git a/etc/aiscript.api.md b/etc/aiscript.api.md index c20a03e2..6ca83a88 100644 --- a/etc/aiscript.api.md +++ b/etc/aiscript.api.md @@ -353,12 +353,14 @@ type FnTypeSource = NodeBase & { type For = NodeBase & { type: 'for'; label?: string; - var?: string; - from?: Expression; - to?: Expression; - times?: Expression; for: Statement | Expression; -}; +} & ({ + var: string; + from: Expression; + to: Expression; +} | { + times: Expression; +}); // @public (undocumented) function getLangVersion(input: string): string | null; @@ -669,7 +671,7 @@ type Return = NodeBase & { // @public (undocumented) export class Scope { - constructor(layerdStates?: Scope['layerdStates'], parent?: Scope, name?: Scope['name'], nsName?: string); + constructor(layeredStates?: Scope['layeredStates'], parent?: Scope, name?: Scope['name'], nsName?: string); add(name: string, variable: Variable): void; assign(name: string, val: Value): void; // (undocumented) diff --git a/src/error.ts b/src/error.ts index 2e1acf8e..5ab0b015 100644 --- a/src/error.ts +++ b/src/error.ts @@ -1,4 +1,3 @@ -import { TokenKind } from './parser/token.js'; import type { Pos } from './node.js'; export abstract class AiScriptError extends Error { @@ -25,9 +24,12 @@ export abstract class AiScriptError extends Error { export class NonAiScriptError extends AiScriptError { public name = 'Internal'; constructor(error: unknown) { - const message = String( - (error as { message?: unknown } | null | undefined)?.message ?? error, - ); + let message: string; + if (error != null && typeof error === 'object' && 'message' in error) { + message = String(error.message); + } else { + message = String(error); + } super(message, error); } } diff --git a/src/interpreter/index.ts b/src/interpreter/index.ts index 8cdab2ae..a4df6679 100644 --- a/src/interpreter/index.ts +++ b/src/interpreter/index.ts @@ -14,7 +14,7 @@ import { getPrimProp } from './primitive-props.js'; import { Variable } from './variable.js'; import { Reference } from './reference.js'; import type { JsValue } from './util.js'; -import type { Value, VFn, VUserFn } from './value.js'; +import type { Value, VFn, VFnParam } from './value.js'; export type LogObject = { scope?: string; @@ -61,7 +61,7 @@ export class Interpreter { const q = args[0]; assertString(q); if (this.opts.in == null) return NULL; - const a = await this.opts.in!(q.value); + const a = await this.opts.in(q.value); return STR(a); }), }; @@ -280,15 +280,19 @@ export class Interpreter { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition return result ?? NULL; } else { - const fnScope = fn.scope!.createChildScope(); + const fnScope = fn.scope.createChildScope(); for (const [i, param] of fn.params.entries()) { - const arg = args[i]; - if (!param.default) expectAny(arg); - this.define(fnScope, param.dest, arg ?? param.default!, true); + let arg = args[i]; + if (!param.default) { + expectAny(arg); + } else if (!arg) { + arg = param.default; + } + this.define(fnScope, param.dest, arg, true); } const info: CallInfo = { name: fn.name ?? '', pos }; - return unWrapRet(await this._run(fn.statements!, fnScope, [...callStack, info])); + return unWrapRet(await this._run(fn.statements, fnScope, [...callStack, info])); } } @@ -430,7 +434,7 @@ export class Interpreter { } case 'for': { - if (node.times) { + if ('times' in node) { const times = await this._eval(node.times, scope, callStack); if (isControl(times)) { return times; @@ -452,11 +456,11 @@ export class Interpreter { } } } else { - const from = await this._eval(node.from!, scope, callStack); + const from = await this._eval(node.from, scope, callStack); if (isControl(from)) { return from; } - const to = await this._eval(node.to!, scope, callStack); + const to = await this._eval(node.to, scope, callStack); if (isControl(to)) { return to; } @@ -464,7 +468,7 @@ export class Interpreter { assertNumber(to); for (let i = from.value; i < from.value + to.value; i++) { const v = await this._eval(node.for, scope.createChildScope(new Map([ - [node.var!, { + [node.var, { isMutable: false, value: NUM(i), }], @@ -632,8 +636,9 @@ export class Interpreter { return target; } if (isObject(target)) { - if (target.value.has(node.name)) { - return target.value.get(node.name)!; + const value = target.value.get(node.name); + if (value != null) { + return value; } else { return NULL; } @@ -660,8 +665,9 @@ export class Interpreter { return item; } else if (isObject(target)) { assertString(i); - if (target.value.has(i.value)) { - return target.value.get(i.value)!; + const value = target.value.get(i.value); + if (value != null) { + return value; } else { return NULL; } @@ -698,28 +704,21 @@ export class Interpreter { } case 'fn': { - const params = await Promise.all(node.params.map(async (param) => { - return { + const params: VFnParam[] = []; + for (const param of node.params) { + const defaultValue = param.default ? await this._eval(param.default, scope, callStack) : + param.optional ? NULL : + undefined; + if (defaultValue != null && isControl(defaultValue)) { + return defaultValue; + } + params.push({ dest: param.dest, - default: - param.default ? await this._eval(param.default, scope, callStack) : - param.optional ? NULL : - undefined, + default: defaultValue, // type: (TODO) - }; - })); - const control = params - .map((param) => param.default) - .filter((value) => value != null) - .find(isControl); - if (control != null) { - return control; + }); } - return FN( - params as VUserFn['params'], - node.children, - scope, - ); + return FN(params, node.children, scope); } case 'block': { @@ -887,9 +886,7 @@ export class Interpreter { let v: Value | Control = NULL; - for (let i = 0; i < program.length; i++) { - const node = program[i]!; - + for (const node of program) { v = await this._eval(node, scope, callStack); if (v.type === 'return') { this.log('block:return', { scope: scope.name, val: v.value }); diff --git a/src/interpreter/primitive-props.ts b/src/interpreter/primitive-props.ts index df3c18cc..6bb61b9e 100644 --- a/src/interpreter/primitive-props.ts +++ b/src/interpreter/primitive-props.ts @@ -9,9 +9,11 @@ import type { Value, VArr, VFn, VNum, VStr, VError } from './value.js'; type VWithPP = VNum|VStr|VArr|VError; const PRIMITIVE_PROPS: { - [key in VWithPP['type']]: { [key: string]: (target: Value) => Value } + [key in VWithPP['type']]: Map Value>; +} & { + [key in (Exclude)['type']]?: never; } = { - num: { + num: new Map(Object.entries({ to_str: (target: VNum): VFn => FN_NATIVE(async (_, _opts) => { return STR(target.value.toString()); }), @@ -19,9 +21,9 @@ const PRIMITIVE_PROPS: { to_hex: (target: VNum): VFn => FN_NATIVE(async (_, _opts) => { return STR(target.value.toString(16)); }), - }, + })), - str: { + str: new Map(Object.entries({ to_num: (target: VStr): VFn => FN_NATIVE(async (_, _opts) => { const parsed = parseInt(target.value, 10); if (isNaN(parsed)) return NULL; @@ -168,9 +170,9 @@ const PRIMITIVE_PROPS: { return STR(target.value.padEnd(width.value, s)); }), - }, + })), - arr: { + arr: new Map(Object.entries({ len: (target: VArr): VNum => NUM(target.value.length), push: (target: VArr): VFn => FN_NATIVE(async ([val], _opts) => { @@ -219,9 +221,8 @@ const PRIMITIVE_PROPS: { filter: (target: VArr): VFn => FN_NATIVE(async ([fn], opts) => { assertFunction(fn); - const vals = [] as Value[]; - for (let i = 0; i < target.value.length; i++) { - const item = target.value[i]!; + const vals: Value[] = []; + for (const [i, item] of target.value.entries()) { const res = await opts.call(fn, [item, NUM(i)]); assertBoolean(res); if (res.value) vals.push(item); @@ -243,8 +244,7 @@ const PRIMITIVE_PROPS: { find: (target: VArr): VFn => FN_NATIVE(async ([fn], opts) => { assertFunction(fn); - for (let i = 0; i < target.value.length; i++) { - const item = target.value[i]!; + for (const [i, item] of target.value.entries()) { const res = await opts.call(fn, [item, NUM(i)]); assertBoolean(res); if (res.value) return item; @@ -382,8 +382,7 @@ const PRIMITIVE_PROPS: { every: (target: VArr): VFn => FN_NATIVE(async ([fn], opts) => { assertFunction(fn); - for (let i = 0; i < target.value.length; i++) { - const item = target.value[i]!; + for (const [i, item] of target.value.entries()) { const res = await opts.call(fn, [item, NUM(i)]); assertBoolean(res); if (!res.value) return FALSE; @@ -393,8 +392,7 @@ const PRIMITIVE_PROPS: { some: (target: VArr): VFn => FN_NATIVE(async ([fn], opts) => { assertFunction(fn); - for (let i = 0; i < target.value.length; i++) { - const item = target.value[i]!; + for (const [i, item] of target.value.entries()) { const res = await opts.call(fn, [item, NUM(i)]); assertBoolean(res); if (res.value) return TRUE; @@ -423,20 +421,21 @@ const PRIMITIVE_PROPS: { assertNumber(index); return target.value.at(index.value) ?? otherwise ?? NULL; }), - }, + })), - error: { + error: new Map(Object.entries({ name: (target: VError): VStr => STR(target.value), info: (target: VError): Value => target.info ?? NULL, - }, + })), } as const; export function getPrimProp(target: Value, name: string): Value { - if (Object.hasOwn(PRIMITIVE_PROPS, target.type)) { - const props = PRIMITIVE_PROPS[target.type as VWithPP['type']]; - if (Object.hasOwn(props, name)) { - return props[name]!(target); + const props = PRIMITIVE_PROPS[target.type]; + if (props != null) { + const prop = props.get(name); + if (prop != null) { + return prop(target); } else { throw new AiScriptRuntimeError(`No such prop (${name}) in ${target.type}.`); } diff --git a/src/interpreter/scope.ts b/src/interpreter/scope.ts index 34142551..c6640c49 100644 --- a/src/interpreter/scope.ts +++ b/src/interpreter/scope.ts @@ -4,9 +4,11 @@ import type { Value } from './value.js'; import type { Variable } from './variable.js'; import type { LogObject } from './index.js'; +export type LayeredStates = [Map, ...Map[]] + export class Scope { private parent?: Scope; - private layerdStates: Map[]; + private layeredStates: LayeredStates; public name: string; public opts: { log?(type: string, params: LogObject): void; @@ -14,10 +16,10 @@ export class Scope { } = {}; public nsName?: string; - constructor(layerdStates: Scope['layerdStates'] = [], parent?: Scope, name?: Scope['name'], nsName?: string) { - this.layerdStates = layerdStates; + constructor(layeredStates: Scope['layeredStates'] = [new Map()], parent?: Scope, name?: Scope['name'], nsName?: string) { + this.layeredStates = layeredStates; this.parent = parent; - this.name = name || (layerdStates.length === 1 ? '' : ''); + this.name = name || (layeredStates.length === 1 ? '' : ''); this.nsName = nsName; } @@ -41,13 +43,13 @@ export class Scope { @autobind public createChildScope(states: Map = new Map(), name?: Scope['name']): Scope { - const layer = [states, ...this.layerdStates]; + const layer: LayeredStates = [states, ...this.layeredStates]; return new Scope(layer, this, name); } @autobind public createChildNamespaceScope(nsName: string, states: Map = new Map(), name?: Scope['name']): Scope { - const layer = [states, ...this.layerdStates]; + const layer: LayeredStates = [states, ...this.layeredStates]; return new Scope(layer, this, name, nsName); } @@ -57,9 +59,10 @@ export class Scope { */ @autobind public get(name: string): Value { - for (const layer of this.layerdStates) { - if (layer.has(name)) { - const state = layer.get(name)!.value; + for (const layer of this.layeredStates) { + const value = layer.get(name); + if (value) { + const state = value.value; this.log('read', { var: name, val: state }); return state; } @@ -67,7 +70,7 @@ export class Scope { throw new AiScriptRuntimeError( `No such variable '${name}' in scope '${this.name}'`, - { scope: this.layerdStates }); + { scope: this.layeredStates }); } /** @@ -85,7 +88,7 @@ export class Scope { */ @autobind public exists(name: string): boolean { - for (const layer of this.layerdStates) { + for (const layer of this.layeredStates) { if (layer.has(name)) { this.log('exists', { var: name }); return true; @@ -101,9 +104,9 @@ export class Scope { */ @autobind public getAll(): Map { - const vars = this.layerdStates.reduce((arr, layer) => { + const vars = this.layeredStates.reduce((arr, layer) => { return [...arr, ...layer]; - }, [] as [string, Variable][]); + }, [] satisfies [string, Variable][]); return new Map(vars); } @@ -115,11 +118,11 @@ export class Scope { @autobind public add(name: string, variable: Variable): void { this.log('add', { var: name, val: variable }); - const states = this.layerdStates[0]!; + const states = this.layeredStates[0]; if (states.has(name)) { throw new AiScriptRuntimeError( `Variable '${name}' already exists in scope '${this.name}'`, - { scope: this.layerdStates }); + { scope: this.layeredStates }); } states.set(name, variable); if (this.parent == null) this.onUpdated(name, variable.value); @@ -134,9 +137,9 @@ export class Scope { @autobind public assign(name: string, val: Value): void { let i = 1; - for (const layer of this.layerdStates) { - if (layer.has(name)) { - const variable = layer.get(name)!; + for (const layer of this.layeredStates) { + const variable = layer.get(name); + if (variable != null) { if (!variable.isMutable) { throw new AiScriptRuntimeError(`Cannot assign to an immutable variable ${name}.`); } @@ -144,7 +147,7 @@ export class Scope { variable.value = val; this.log('assign', { var: name, val: val }); - if (i === this.layerdStates.length) this.onUpdated(name, val); + if (i === this.layeredStates.length) this.onUpdated(name, val); return; } i++; @@ -152,6 +155,6 @@ export class Scope { throw new AiScriptRuntimeError( `No such variable '${name}' in scope '${this.name}'`, - { scope: this.layerdStates }); + { scope: this.layeredStates }); } } diff --git a/src/node.ts b/src/node.ts index 1651e849..ae2048d6 100644 --- a/src/node.ts +++ b/src/node.ts @@ -82,12 +82,14 @@ export type Each = NodeBase & { export type For = NodeBase & { type: 'for'; // for文 label?: string; // ラベル - var?: string; // イテレータ変数名 - from?: Expression; // 開始値 - to?: Expression; // 終値 - times?: Expression; // 回数 for: Statement | Expression; // 本体処理 -}; +} & ({ + var: string; // イテレータ変数名 + from: Expression; // 開始値 + to: Expression; // 終値 +} | { + times: Expression; // 回数 +}); export type Loop = NodeBase & { type: 'loop'; // loop文 diff --git a/src/parser/plugins/validate-jump-statements.ts b/src/parser/plugins/validate-jump-statements.ts index 9828bca9..7b4264ff 100644 --- a/src/parser/plugins/validate-jump-statements.ts +++ b/src/parser/plugins/validate-jump-statements.ts @@ -30,7 +30,7 @@ function getCorrespondingBlock(ancestors: Ast.Node[], label?: string): Ast.Each return; } -function validateNode(node: Ast.Node, ancestors: Ast.Node[]): Ast.Node { +function validateNode(node: T, ancestors: Ast.Node[]): T { switch (node.type) { case 'return': { if (!ancestors.some(({ type }) => type === 'fn')) { @@ -55,8 +55,10 @@ function validateNode(node: Ast.Node, ancestors: Ast.Node[]): Ast.Node { break; } case 'for': { - if (block.times != null && ancestors.includes(block.times)) { - throw new AiScriptSyntaxError('break corresponding to for is not allowed in the count', node.loc.start); + if ('times' in block) { + if (ancestors.includes(block.times)) { + throw new AiScriptSyntaxError('break corresponding to for is not allowed in the count', node.loc.start); + } } else if (ancestors.some((ancestor) => ancestor === block.from || ancestor === block.to)) { throw new AiScriptSyntaxError('break corresponding to for is not allowed in the range', node.loc.start); } @@ -107,8 +109,10 @@ function validateNode(node: Ast.Node, ancestors: Ast.Node[]): Ast.Node { break; } case 'for': { - if (block.times != null && ancestors.includes(block.times)) { - throw new AiScriptSyntaxError('continue corresponding to for is not allowed in the count', node.loc.start); + if ('times' in block) { + if (ancestors.includes(block.times)) { + throw new AiScriptSyntaxError('continue corresponding to for is not allowed in the count', node.loc.start); + } } else if (ancestors.some((ancestor) => ancestor === block.from || ancestor === block.to)) { throw new AiScriptSyntaxError('continue corresponding to for is not allowed in the range', node.loc.start); } diff --git a/src/parser/plugins/validate-keyword.ts b/src/parser/plugins/validate-keyword.ts index c134232c..a68ad9e5 100644 --- a/src/parser/plugins/validate-keyword.ts +++ b/src/parser/plugins/validate-keyword.ts @@ -87,7 +87,7 @@ function validateTypeParams(node: Ast.Fn | Ast.FnTypeSource): void { } } -function validateNode(node: Ast.Node): Ast.Node { +function validateNode(node: T): T { switch (node.type) { case 'def': { validateDest(node.dest); @@ -119,7 +119,7 @@ function validateNode(node: Ast.Node): Ast.Node { if (node.label != null && reservedWord.includes(node.label)) { throwReservedWordError(node.label, node.loc); } - if (node.var != null && reservedWord.includes(node.var)) { + if ('var' in node && reservedWord.includes(node.var)) { throwReservedWordError(node.var, node.loc); } break; diff --git a/src/parser/plugins/validate-type.ts b/src/parser/plugins/validate-type.ts index 1d043919..ba613c31 100644 --- a/src/parser/plugins/validate-type.ts +++ b/src/parser/plugins/validate-type.ts @@ -23,7 +23,7 @@ function collectTypeParams(node: Ast.Node, ancestors: Ast.Node[]): Ast.TypeParam return items; } -function validateNode(node: Ast.Node, ancestors: Ast.Node[]): Ast.Node { +function validateNode(node: T, ancestors: Ast.Node[]): T { switch (node.type) { case 'def': { if (node.varType != null) { diff --git a/src/parser/scanner.ts b/src/parser/scanner.ts index cb86984f..9050f176 100644 --- a/src/parser/scanner.ts +++ b/src/parser/scanner.ts @@ -4,7 +4,7 @@ import { TOKEN, TokenKind } from './token.js'; import { unexpectedTokenError } from './utils.js'; import type { ITokenStream } from './streams/token-stream.js'; -import type { Token, TokenPosition } from './token.js'; +import type { IdentifierOrLiteralToken, NormalToken, SimpleToken, TemplateExprToken, TemplateToken, TokenPosition } from './token.js'; const spaceChars = [' ', '\t']; const lineBreakChars = ['\r', '\n']; @@ -14,9 +14,9 @@ const wordChar = /^[A-Za-z0-9_]$/; /** * 入力文字列からトークンを読み取るクラス */ -export class Scanner implements ITokenStream { +export class Scanner implements ITokenStream { private stream: CharStream; - private _tokens: Token[] = []; + private _tokens: [NormalToken, ...NormalToken[]]; constructor(source: string) constructor(stream: CharStream) @@ -26,14 +26,14 @@ export class Scanner implements ITokenStream { } else { this.stream = x; } - this._tokens.push(this.readToken()); + this._tokens = [this.readToken()]; } /** * カーソル位置にあるトークンを取得します。 */ - public getToken(): Token { - return this._tokens[0]!; + public getToken(): NormalToken { + return this._tokens[0]; } /** @@ -46,17 +46,10 @@ export class Scanner implements ITokenStream { /** * カーソル位置にあるトークンの種類を取得します。 */ - public getTokenKind(): TokenKind { + public getTokenKind(): NormalToken['kind'] { return this.getToken().kind; } - /** - * カーソル位置にあるトークンに含まれる値を取得します。 - */ - public getTokenValue(): string { - return this.getToken().value!; - } - /** * カーソル位置にあるトークンの位置情報を取得します。 */ @@ -69,7 +62,7 @@ export class Scanner implements ITokenStream { */ public next(): void { // 現在のトークンがEOFだったら次のトークンに進まない - if (this._tokens[0]!.kind === TokenKind.EOF) { + if (this._tokens[0].kind === TokenKind.EOF) { return; } @@ -83,7 +76,7 @@ export class Scanner implements ITokenStream { /** * トークンの先読みを行います。カーソル位置は移動されません。 */ - public lookahead(offset: number): Token { + public lookahead(offset: number): NormalToken { while (this._tokens.length <= offset) { this._tokens.push(this.readToken()); } @@ -95,21 +88,21 @@ export class Scanner implements ITokenStream { * カーソル位置にあるトークンの種類が指定したトークンの種類と一致することを確認します。 * 一致しなかった場合には文法エラーを発生させます。 */ - public expect(kind: TokenKind): void { + public expect(kind: NormalToken['kind']): void { if (!this.is(kind)) { throw unexpectedTokenError(this.getTokenKind(), this.getPos()); } } - private readToken(): Token { + private readToken(): NormalToken { let hasLeftSpacing = false; while (true) { - if (this.stream.eof) { + if (this.stream.eof()) { return TOKEN(TokenKind.EOF, this.stream.getPos(), { hasLeftSpacing }); } // skip spasing - if (spaceChars.includes(this.stream.char)) { + if (spaceChars.includes(this.stream.char())) { this.stream.next(); hasLeftSpacing = true; continue; @@ -118,17 +111,17 @@ export class Scanner implements ITokenStream { // トークン位置を記憶 const pos = this.stream.getPos(); - if (lineBreakChars.includes(this.stream.char)) { + if (lineBreakChars.includes(this.stream.char())) { this.skipEmptyLines(); return TOKEN(TokenKind.NewLine, pos, { hasLeftSpacing }); } // noFallthroughCasesInSwitchと関数の返り値の型を利用し、全ての場合分けがreturnかcontinueで適切に処理されることを強制している // その都合上、break文の使用ないしこのswitch文の後に処理を書くことは極力避けてほしい - switch (this.stream.char) { + switch (this.stream.char()) { case '!': { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '=') { + if (!this.stream.eof() && (this.stream.char()) === '=') { this.stream.next(); return TOKEN(TokenKind.NotEq, pos, { hasLeftSpacing }); } else { @@ -141,15 +134,15 @@ export class Scanner implements ITokenStream { } case '#': { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '#') { + if (!this.stream.eof() && (this.stream.char()) === '#') { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '#') { + if (!this.stream.eof() && (this.stream.char()) === '#') { this.stream.next(); return TOKEN(TokenKind.Sharp3, pos, { hasLeftSpacing }); } else { throw new AiScriptSyntaxError('invalid sequence of characters: "##"', pos); } - } else if (!this.stream.eof && (this.stream.char as string) === '[') { + } else if (!this.stream.eof() && (this.stream.char()) === '[') { this.stream.next(); return TOKEN(TokenKind.OpenSharpBracket, pos, { hasLeftSpacing }); } else { @@ -162,7 +155,7 @@ export class Scanner implements ITokenStream { } case '&': { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '&') { + if (!this.stream.eof() && (this.stream.char()) === '&') { this.stream.next(); return TOKEN(TokenKind.And2, pos, { hasLeftSpacing }); } else { @@ -183,7 +176,7 @@ export class Scanner implements ITokenStream { } case '+': { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '=') { + if (!this.stream.eof() && (this.stream.char()) === '=') { this.stream.next(); return TOKEN(TokenKind.PlusEq, pos, { hasLeftSpacing }); } else { @@ -196,7 +189,7 @@ export class Scanner implements ITokenStream { } case '-': { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '=') { + if (!this.stream.eof() && (this.stream.char()) === '=') { this.stream.next(); return TOKEN(TokenKind.MinusEq, pos, { hasLeftSpacing }); } else { @@ -209,11 +202,11 @@ export class Scanner implements ITokenStream { } case '/': { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '*') { + if (!this.stream.eof() && (this.stream.char()) === '*') { this.stream.next(); this.skipCommentRange(); continue; - } else if (!this.stream.eof && (this.stream.char as string) === '/') { + } else if (!this.stream.eof() && (this.stream.char()) === '/') { this.stream.next(); this.skipCommentLine(); continue; @@ -223,7 +216,7 @@ export class Scanner implements ITokenStream { } case ':': { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === ':') { + if (!this.stream.eof() && (this.stream.char()) === ':') { this.stream.next(); return TOKEN(TokenKind.Colon2, pos, { hasLeftSpacing }); } else { @@ -236,10 +229,10 @@ export class Scanner implements ITokenStream { } case '<': { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '=') { + if (!this.stream.eof() && (this.stream.char()) === '=') { this.stream.next(); return TOKEN(TokenKind.LtEq, pos, { hasLeftSpacing }); - } else if (!this.stream.eof && (this.stream.char as string) === ':') { + } else if (!this.stream.eof() && (this.stream.char()) === ':') { this.stream.next(); return TOKEN(TokenKind.Out, pos, { hasLeftSpacing }); } else { @@ -248,10 +241,10 @@ export class Scanner implements ITokenStream { } case '=': { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '=') { + if (!this.stream.eof() && (this.stream.char()) === '=') { this.stream.next(); return TOKEN(TokenKind.Eq2, pos, { hasLeftSpacing }); - } else if (!this.stream.eof && (this.stream.char as string) === '>') { + } else if (!this.stream.eof() && (this.stream.char()) === '>') { this.stream.next(); return TOKEN(TokenKind.Arrow, pos, { hasLeftSpacing }); } else { @@ -260,7 +253,7 @@ export class Scanner implements ITokenStream { } case '>': { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '=') { + if (!this.stream.eof() && (this.stream.char()) === '=') { this.stream.next(); return TOKEN(TokenKind.GtEq, pos, { hasLeftSpacing }); } else { @@ -300,7 +293,7 @@ export class Scanner implements ITokenStream { } case '|': { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '|') { + if (!this.stream.eof() && (this.stream.char()) === '|') { this.stream.next(); return TOKEN(TokenKind.Or2, pos, { hasLeftSpacing }); } else { @@ -318,7 +311,7 @@ export class Scanner implements ITokenStream { const wordToken = this.tryReadWord(hasLeftSpacing); if (wordToken) return wordToken; - throw new AiScriptSyntaxError(`invalid character: "${this.stream.char}"`, pos); + throw new AiScriptSyntaxError(`invalid character: "${this.stream.char()}"`, pos); } } // Use `return` or `continue` before reaching this line. @@ -329,14 +322,14 @@ export class Scanner implements ITokenStream { // Do not add any more code here. This line should be unreachable. } - private tryReadWord(hasLeftSpacing: boolean): Token | undefined { + private tryReadWord(hasLeftSpacing: boolean): SimpleToken | IdentifierOrLiteralToken | undefined { // read a word let value = ''; const pos = this.stream.getPos(); - while (!this.stream.eof && wordChar.test(this.stream.char)) { - value += this.stream.char; + while (!this.stream.eof() && wordChar.test(this.stream.char())) { + value += this.stream.char(); this.stream.next(); } if (value.length === 0) { @@ -413,23 +406,23 @@ export class Scanner implements ITokenStream { } } - private tryReadDigits(hasLeftSpacing: boolean): Token | undefined { + private tryReadDigits(hasLeftSpacing: boolean): IdentifierOrLiteralToken | undefined { let wholeNumber = ''; let fractional = ''; const pos = this.stream.getPos(); - while (!this.stream.eof && digit.test(this.stream.char)) { - wholeNumber += this.stream.char; + while (!this.stream.eof() && digit.test(this.stream.char())) { + wholeNumber += this.stream.char(); this.stream.next(); } if (wholeNumber.length === 0) { return; } - if (!this.stream.eof && this.stream.char === '.') { + if (!this.stream.eof() && this.stream.char() === '.') { this.stream.next(); - while (!this.stream.eof as boolean && digit.test(this.stream.char as string)) { - fractional += this.stream.char; + while (!this.stream.eof() && digit.test(this.stream.char())) { + fractional += this.stream.char(); this.stream.next(); } if (fractional.length === 0) { @@ -445,9 +438,9 @@ export class Scanner implements ITokenStream { return TOKEN(TokenKind.NumberLiteral, pos, { hasLeftSpacing, value }); } - private readStringLiteral(hasLeftSpacing: boolean): Token { + private readStringLiteral(hasLeftSpacing: boolean): IdentifierOrLiteralToken { let value = ''; - const literalMark = this.stream.char; + const literalMark = this.stream.char(); let state: 'string' | 'escape' | 'finish' = 'string'; const pos = this.stream.getPos(); @@ -456,28 +449,28 @@ export class Scanner implements ITokenStream { while (state !== 'finish') { switch (state) { case 'string': { - if (this.stream.eof) { + if (this.stream.eof()) { throw new AiScriptUnexpectedEOFError(pos); } - if (this.stream.char === '\\') { + if (this.stream.char() === '\\') { this.stream.next(); state = 'escape'; break; } - if (this.stream.char === literalMark) { + if (this.stream.char() === literalMark) { this.stream.next(); state = 'finish'; break; } - value += this.stream.char; + value += this.stream.char(); this.stream.next(); break; } case 'escape': { - if (this.stream.eof) { + if (this.stream.eof()) { throw new AiScriptUnexpectedEOFError(pos); } - value += this.stream.char; + value += this.stream.char(); this.stream.next(); state = 'string'; break; @@ -487,10 +480,10 @@ export class Scanner implements ITokenStream { return TOKEN(TokenKind.StringLiteral, pos, { hasLeftSpacing, value }); } - private readTemplate(hasLeftSpacing: boolean): Token { - const elements: Token[] = []; + private readTemplate(hasLeftSpacing: boolean): TemplateToken { + const elements: TemplateToken['children'] = []; let buf = ''; - let tokenBuf: Token[] = []; + let tokenBuf: TemplateExprToken['children'] = []; let state: 'string' | 'escape' | 'expr' | 'finish' = 'string'; let exprBracketDepth = 0; @@ -502,17 +495,17 @@ export class Scanner implements ITokenStream { switch (state) { case 'string': { // テンプレートの終了が無いままEOFに達した - if (this.stream.eof) { + if (this.stream.eof()) { throw new AiScriptUnexpectedEOFError(pos); } // エスケープ - if (this.stream.char === '\\') { + if (this.stream.char() === '\\') { this.stream.next(); state = 'escape'; break; } // テンプレートの終了 - if (this.stream.char === '`') { + if (this.stream.char() === '`') { this.stream.next(); if (buf.length > 0) { elements.push(TOKEN(TokenKind.TemplateStringElement, elementPos, { hasLeftSpacing, value: buf })); @@ -521,7 +514,7 @@ export class Scanner implements ITokenStream { break; } // 埋め込み式の開始 - if (this.stream.char === '{') { + if (this.stream.char() === '{') { this.stream.next(); if (buf.length > 0) { elements.push(TOKEN(TokenKind.TemplateStringElement, elementPos, { hasLeftSpacing, value: buf })); @@ -532,17 +525,17 @@ export class Scanner implements ITokenStream { state = 'expr'; break; } - buf += this.stream.char; + buf += this.stream.char(); this.stream.next(); break; } case 'escape': { // エスケープ対象の文字が無いままEOFに達した - if (this.stream.eof) { + if (this.stream.eof()) { throw new AiScriptUnexpectedEOFError(pos); } // 普通の文字として取り込み - buf += this.stream.char; + buf += this.stream.char(); this.stream.next(); // 通常の文字列に戻る state = 'string'; @@ -550,18 +543,18 @@ export class Scanner implements ITokenStream { } case 'expr': { // 埋め込み式の終端記号が無いままEOFに達した - if (this.stream.eof) { + if (this.stream.eof()) { throw new AiScriptUnexpectedEOFError(pos); } // skip spasing - if (spaceChars.includes(this.stream.char)) { + if (spaceChars.includes(this.stream.char())) { this.stream.next(); continue; } - if (this.stream.char === '{') { + if (this.stream.char() === '{') { exprBracketDepth++; } - if ((this.stream.char as string) === '}') { + if ((this.stream.char()) === '}') { // 埋め込み式の終了 if (exprBracketDepth === 0) { elements.push(TOKEN(TokenKind.TemplateExprElement, elementPos, { hasLeftSpacing, children: tokenBuf })); @@ -587,20 +580,20 @@ export class Scanner implements ITokenStream { } private skipEmptyLines(): void { - while (!this.stream.eof) { + while (!this.stream.eof()) { // skip spacing - if (spaceChars.includes(this.stream.char) || lineBreakChars.includes(this.stream.char)) { + if (spaceChars.includes(this.stream.char()) || lineBreakChars.includes(this.stream.char())) { this.stream.next(); continue; } - if (this.stream.char === '/') { + if (this.stream.char() === '/') { this.stream.next(); - if (!this.stream.eof && (this.stream.char as string) === '*') { + if (!this.stream.eof() && (this.stream.char()) === '*') { this.stream.next(); this.skipCommentRange(); continue; - } else if (!this.stream.eof && (this.stream.char as string) === '/') { + } else if (!this.stream.eof() && (this.stream.char()) === '/') { this.stream.next(); this.skipCommentLine(); continue; @@ -615,10 +608,10 @@ export class Scanner implements ITokenStream { private skipCommentLine(): void { while (true) { - if (this.stream.eof) { + if (this.stream.eof()) { break; } - if (this.stream.char === '\n') { + if (this.stream.char() === '\n') { break; } this.stream.next(); @@ -627,15 +620,15 @@ export class Scanner implements ITokenStream { private skipCommentRange(): void { while (true) { - if (this.stream.eof) { + if (this.stream.eof()) { throw new AiScriptUnexpectedEOFError(this.stream.getPos()); } - if (this.stream.char === '*') { + if (this.stream.char() === '*') { this.stream.next(); - if (this.stream.eof) { + if (this.stream.eof()) { throw new AiScriptUnexpectedEOFError(this.stream.getPos()); } - if ((this.stream.char as string) === '/') { + if ((this.stream.char()) === '/') { this.stream.next(); break; } diff --git a/src/parser/streams/char-stream.ts b/src/parser/streams/char-stream.ts index 58b36793..0bc7f8b3 100644 --- a/src/parser/streams/char-stream.ts +++ b/src/parser/streams/char-stream.ts @@ -28,15 +28,15 @@ export class CharStream { /** * ストリームの終わりに達しているかどうかを取得します。 */ - public get eof(): boolean { + public eof(): boolean { return this.endOfPage && this.isLastPage; } /** * カーソル位置にある文字を取得します。 */ - public get char(): string { - if (this.eof) { + public char(): string { + if (this.eof()) { throw new Error('end of stream'); } return this._char!; @@ -56,7 +56,7 @@ export class CharStream { * カーソル位置を次の文字へ進めます。 */ public next(): void { - if (!this.eof && this._char === '\n') { + if (!this.eof() && this._char === '\n') { this.line++; this.column = 0; } else { @@ -90,7 +90,7 @@ export class CharStream { private moveNext(): void { this.loadChar(); while (true) { - if (!this.eof && this._char === '\r') { + if (!this.eof() && this._char === '\r') { this.incAddr(); this.loadChar(); continue; @@ -111,7 +111,7 @@ export class CharStream { private movePrev(): void { this.loadChar(); while (true) { - if (!this.eof && this._char === '\r') { + if (!this.eof() && this._char === '\r') { this.decAddr(); this.loadChar(); continue; @@ -130,7 +130,7 @@ export class CharStream { } private loadChar(): void { - if (this.eof) { + if (this.eof()) { this._char = undefined; } else { this._char = this.pages.get(this.pageIndex)![this.address]!; diff --git a/src/parser/streams/token-stream.ts b/src/parser/streams/token-stream.ts index e21b66a8..b31190eb 100644 --- a/src/parser/streams/token-stream.ts +++ b/src/parser/streams/token-stream.ts @@ -5,26 +5,21 @@ import type { Token, TokenPosition } from '../token.js'; /** * トークンの読み取りに関するインターフェース */ -export interface ITokenStream { +export interface ITokenStream { /** * カーソル位置にあるトークンを取得します。 */ - getToken(): Token; + getToken(): T; /** * カーソル位置にあるトークンの種類が指定したトークンの種類と一致するかどうかを示す値を取得します。 */ - is(kind: TokenKind): boolean; + is(kind: T['kind']): boolean; /** * カーソル位置にあるトークンの種類を取得します。 */ - getTokenKind(): TokenKind; - - /** - * カーソル位置にあるトークンに含まれる値を取得します。 - */ - getTokenValue(): string; + getTokenKind(): T['kind']; /** * カーソル位置にあるトークンの位置情報を取得します。 @@ -39,13 +34,13 @@ export interface ITokenStream { /** * トークンの先読みを行います。カーソル位置は移動されません。 */ - lookahead(offset: number): Token; + lookahead(offset: number): T; /** * カーソル位置にあるトークンの種類が指定したトークンの種類と一致することを確認します。 * 一致しなかった場合には文法エラーを発生させます。 */ - expect(kind: TokenKind): void; + expect(kind: T['kind']): void; } /** @@ -83,13 +78,6 @@ export class TokenStream implements ITokenStream { return this.getTokenKind() === kind; } - /** - * カーソル位置にあるトークンに含まれる値を取得します。 - */ - public getTokenValue(): string { - return this.getToken().value!; - } - /** * カーソル位置にあるトークンの種類を取得します。 */ diff --git a/src/parser/syntaxes/common.ts b/src/parser/syntaxes/common.ts index 049f8553..c69a0c4f 100644 --- a/src/parser/syntaxes/common.ts +++ b/src/parser/syntaxes/common.ts @@ -1,4 +1,4 @@ -import { TokenKind } from '../token.js'; +import { expectTokenKind, TokenKind } from '../token.js'; import { AiScriptSyntaxError, AiScriptUnexpectedEOFError } from '../../error.js'; import { NODE } from '../utils.js'; import { parseStatement } from './statements.js'; @@ -15,9 +15,10 @@ import type * as Ast from '../../node.js'; */ export function parseDest(s: ITokenStream): Ast.Expression { // 全部parseExprに任せるとparseReferenceが型注釈を巻き込んでパースしてしまうためIdentifierのみ個別に処理。 - if (s.is(TokenKind.Identifier)) { - const nameStartPos = s.getPos(); - const name = s.getTokenValue(); + const token = s.getToken(); + if (token.kind === TokenKind.Identifier) { + const nameStartPos = token.pos; + const name = token.value; s.next(); return NODE('identifier', { name }, nameStartPos, s.getPos()); } else { @@ -148,7 +149,9 @@ export function parseLabel(s: ITokenStream): string { throw new AiScriptSyntaxError('cannot use spaces in a label', s.getPos()); } s.expect(TokenKind.Identifier); - const label = s.getTokenValue(); + const token = s.getToken(); + expectTokenKind(token, TokenKind.Identifier); + const label = token.value; s.next(); return label; diff --git a/src/parser/syntaxes/expressions.ts b/src/parser/syntaxes/expressions.ts index e249548b..a1080469 100644 --- a/src/parser/syntaxes/expressions.ts +++ b/src/parser/syntaxes/expressions.ts @@ -1,7 +1,7 @@ import { AiScriptSyntaxError, AiScriptUnexpectedEOFError } from '../../error.js'; import { NODE, unexpectedTokenError } from '../utils.js'; import { TokenStream } from '../streams/token-stream.js'; -import { TokenKind } from '../token.js'; +import { expectTokenKind, TokenKind } from '../token.js'; import { parseBlock, parseLabel, parseOptionalSeparator, parseParams } from './common.js'; import { parseBlockOrStatement } from './statements.js'; import { parseType, parseTypeParams } from './types.js'; @@ -103,8 +103,9 @@ function parseInfix(s: ITokenStream, left: Ast.Expression, minBp: number): Ast.E } if (op === TokenKind.Dot) { - s.expect(TokenKind.Identifier); - const name = s.getTokenValue(); + const token = s.getToken(); + expectTokenKind(token, TokenKind.Identifier); + const name = token.value; s.next(); return NODE('prop', { @@ -192,8 +193,9 @@ function parsePostfix(s: ITokenStream, expr: Ast.Expression): Ast.Expression { function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Expression { const startPos = s.getPos(); + const token = s.getToken(); - switch (s.getTokenKind()) { + switch (token.kind) { case TokenKind.IfKeyword: { if (isStatic) break; return parseIf(s); @@ -219,12 +221,12 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Expression { if (isStatic) break; - for (const [i, element] of s.getToken().children!.entries()) { + for (const [i, element] of token.children.entries()) { switch (element.kind) { case TokenKind.TemplateStringElement: { // トークンの終了位置を取得するために先読み - const nextToken = s.getToken().children![i + 1] ?? s.lookahead(1); - values.push(NODE('str', { value: element.value! }, element.pos, nextToken.pos)); + const nextToken = token.children[i + 1] ?? s.lookahead(1); + values.push(NODE('str', { value: element.value }, element.pos, nextToken.pos)); break; } case TokenKind.TemplateExprElement: { @@ -251,13 +253,13 @@ function parseAtom(s: ITokenStream, isStatic: boolean): Ast.Expression { return NODE('tmpl', { tmpl: values }, startPos, s.getPos()); } case TokenKind.StringLiteral: { - const value = s.getTokenValue(); + const value = token.value; s.next(); return NODE('str', { value }, startPos, s.getPos()); } case TokenKind.NumberLiteral: { // TODO: validate number value - const value = Number(s.getTokenValue()); + const value = Number(token.value); s.next(); return NODE('num', { value }, startPos, s.getPos()); } @@ -568,8 +570,9 @@ function parseReference(s: ITokenStream): Ast.Identifier { break; } } - s.expect(TokenKind.Identifier); - segs.push(s.getTokenValue()); + const token = s.getToken(); + expectTokenKind(token, TokenKind.Identifier); + segs.push(token.value); s.next(); } return NODE('identifier', { name: segs.join(':') }, startPos, s.getPos()); @@ -592,11 +595,11 @@ function parseObject(s: ITokenStream, isStatic: boolean): Ast.Obj { const map = new Map(); while (!s.is(TokenKind.CloseBrace)) { - const keyTokenKind = s.getTokenKind(); - if (keyTokenKind !== TokenKind.Identifier && keyTokenKind !== TokenKind.StringLiteral) { - throw unexpectedTokenError(keyTokenKind, s.getPos()); + const token = s.getToken(); + if (token.kind !== TokenKind.Identifier && token.kind !== TokenKind.StringLiteral) { + throw unexpectedTokenError(token.kind, s.getPos()); } - const k = s.getTokenValue(); + const k = token.value; s.next(); s.expect(TokenKind.Colon); diff --git a/src/parser/syntaxes/statements.ts b/src/parser/syntaxes/statements.ts index d8918adf..2e6a6284 100644 --- a/src/parser/syntaxes/statements.ts +++ b/src/parser/syntaxes/statements.ts @@ -1,6 +1,6 @@ import { AiScriptSyntaxError } from '../../error.js'; -import { CALL_NODE, NODE, unexpectedTokenError } from '../utils.js'; -import { TokenKind } from '../token.js'; +import { CALL_NODE, LOC, NODE, unexpectedTokenError } from '../utils.js'; +import { expectTokenKind, TokenKind } from '../token.js'; import { parseBlock, parseDest, parseLabel, parseParams } from './common.js'; import { parseExpr } from './expressions.js'; import { parseType, parseTypeParams } from './types.js'; @@ -155,9 +155,10 @@ function parseFnDef(s: ITokenStream): Ast.Definition { s.expect(TokenKind.At); s.next(); - s.expect(TokenKind.Identifier); + const token = s.getToken(); + expectTokenKind(token, TokenKind.Identifier); const nameStartPos = s.getPos(); - const name = s.getTokenValue(); + const name = token.value; s.next(); const dest = NODE('identifier', { name }, nameStartPos, s.getPos()); @@ -307,8 +308,9 @@ function parseFor(s: ITokenStream): Ast.For { const identPos = s.getPos(); - s.expect(TokenKind.Identifier); - const name = s.getTokenValue(); + const token = s.getToken(); + expectTokenKind(token, TokenKind.Identifier); + const name = token.value; s.next(); let _from: Ast.Expression; @@ -334,12 +336,14 @@ function parseFor(s: ITokenStream): Ast.For { const body = parseBlockOrStatement(s); - return NODE('for', { + return { + type: 'for', var: name, from: _from, to, for: body, - }, startPos, s.getPos()); + loc: LOC(startPos, s.getPos()), + }; } else { // times syntax @@ -352,10 +356,12 @@ function parseFor(s: ITokenStream): Ast.For { const body = parseBlockOrStatement(s); - return NODE('for', { + return { + type: 'for', times, for: body, - }, startPos, s.getPos()); + loc: LOC(startPos, s.getPos()), + }; } } @@ -412,8 +418,9 @@ function parseAttr(s: ITokenStream): Ast.Attribute { s.expect(TokenKind.OpenSharpBracket); s.next(); - s.expect(TokenKind.Identifier); - const name = s.getTokenValue(); + const token = s.getToken(); + expectTokenKind(token, TokenKind.Identifier); + const name = token.value; s.next(); let value: Ast.Expression; diff --git a/src/parser/syntaxes/toplevel.ts b/src/parser/syntaxes/toplevel.ts index 91a07787..e8060289 100644 --- a/src/parser/syntaxes/toplevel.ts +++ b/src/parser/syntaxes/toplevel.ts @@ -1,5 +1,5 @@ import { NODE } from '../utils.js'; -import { TokenKind } from '../token.js'; +import { expectTokenKind, TokenKind } from '../token.js'; import { AiScriptSyntaxError, AiScriptUnexpectedEOFError } from '../../error.js'; import { parseDefStatement, parseStatement } from './statements.js'; import { parseExpr } from './expressions.js'; @@ -67,8 +67,9 @@ export function parseNamespace(s: ITokenStream): Ast.Namespace { s.expect(TokenKind.Colon2); s.next(); - s.expect(TokenKind.Identifier); - const name = s.getTokenValue(); + const token = s.getToken(); + expectTokenKind(token, TokenKind.Identifier); + const name = token.value; s.next(); const members: (Ast.Namespace | Ast.Definition)[] = []; @@ -131,8 +132,9 @@ export function parseMeta(s: ITokenStream): Ast.Meta { s.next(); let name = null; - if (s.is(TokenKind.Identifier)) { - name = s.getTokenValue(); + const token = s.getToken(); + if (token.kind === TokenKind.Identifier) { + name = token.value; s.next(); } diff --git a/src/parser/syntaxes/types.ts b/src/parser/syntaxes/types.ts index a29ced1a..dc0258cb 100644 --- a/src/parser/syntaxes/types.ts +++ b/src/parser/syntaxes/types.ts @@ -1,5 +1,5 @@ import { AiScriptSyntaxError, AiScriptUnexpectedEOFError } from '../../error.js'; -import { TokenKind } from '../token.js'; +import { expectTokenKind, TokenKind } from '../token.js'; import { NODE } from '../utils.js'; import { parseOptionalSeparator } from './common.js'; @@ -51,8 +51,9 @@ export function parseTypeParams(s: ITokenStream): TypeParam[] { * ``` */ function parseTypeParam(s: ITokenStream): TypeParam { - s.expect(TokenKind.Identifier); - const name = s.getTokenValue(); + const token = s.getToken(); + expectTokenKind(token, TokenKind.Identifier); + const name = token.value; s.next(); return { name }; @@ -154,8 +155,9 @@ function parseNamedType(s: ITokenStream): Ast.TypeSource { const startPos = s.getPos(); let name: string; - if (s.is(TokenKind.Identifier)) { - name = s.getTokenValue(); + const token = s.getToken(); + if (token.kind === TokenKind.Identifier) { + name = token.value; s.next(); } else { s.expect(TokenKind.NullKeyword); diff --git a/src/parser/token.ts b/src/parser/token.ts index 35b2d4c7..d5b50a4f 100644 --- a/src/parser/token.ts +++ b/src/parser/token.ts @@ -1,3 +1,5 @@ +import { unexpectedTokenError } from './utils.js'; + export enum TokenKind { EOF, NewLine, @@ -115,22 +117,99 @@ export enum TokenKind { export type TokenPosition = { column: number, line: number }; -export class Token { - constructor( - public kind: TokenKind, - public pos: TokenPosition, - public hasLeftSpacing = false, - /** for number literal, string literal */ - public value?: string, - /** for template syntax */ - public children?: Token[], - ) { } +type TokenBase = { + kind: TokenKind; + pos: TokenPosition; + hasLeftSpacing: boolean; +}; + +export type EOFToken = TokenBase & { + kind: TokenKind.EOF; +}; + +export type SimpleToken = TokenBase & { + kind: Exclude< + TokenKind, + TokenKind.EOF + | TokenKind.Identifier + | TokenKind.NumberLiteral + | TokenKind.StringLiteral + | TokenKind.Template + | TokenKind.TemplateStringElement + | TokenKind.TemplateExprElement + >; +} + +/** for number literal, string literal */ +export type IdentifierOrLiteralToken = TokenBase & { + kind: TokenKind.Identifier | TokenKind.NumberLiteral | TokenKind.StringLiteral; + value: string; +}; + +/** for template syntax */ +export type TemplateToken = TokenBase & { + kind: TokenKind.Template; + children: (EOFToken | TemplateExprToken | TemplateStringToken)[]; +}; + +export type TemplateStringToken = TokenBase & { + kind: TokenKind.TemplateStringElement; + value: string; +}; + +export type TemplateExprToken = TokenBase & { + kind: TokenKind.TemplateExprElement; + children: NormalToken[]; +}; + +export type NormalToken = EOFToken | SimpleToken | IdentifierOrLiteralToken | TemplateToken; + +export type Token = NormalToken | TemplateStringToken | TemplateExprToken; + +export function TOKEN( + kind: EOFToken['kind'], + pos: TokenPosition, + opts?: Omit, +): EOFToken; +export function TOKEN( + kind: SimpleToken['kind'], + pos: TokenPosition, + opts: Omit, +): SimpleToken; +export function TOKEN( + kind: IdentifierOrLiteralToken['kind'], + pos: TokenPosition, + opts: Omit, +): IdentifierOrLiteralToken; +export function TOKEN( + kind: TemplateToken['kind'], + pos: TokenPosition, + opts: Omit, +): TemplateToken; +export function TOKEN( + kind: TemplateStringToken['kind'], + pos: TokenPosition, + opts: Omit, +): TemplateStringToken; +export function TOKEN( + kind: TemplateExprToken['kind'], + pos: TokenPosition, + opts: Omit, +): TemplateExprToken; +export function TOKEN( + kind: TokenBase['kind'], + pos: TokenPosition, + opts?: Omit, +): TokenBase { + if (opts == null) { + return { kind, pos, hasLeftSpacing: false }; + } else { + return { kind, pos, ...opts }; + } } -/** - * - opts.value: for number literal, string literal - * - opts.children: for template syntax -*/ -export function TOKEN(kind: TokenKind, pos: TokenPosition, opts?: { hasLeftSpacing?: boolean, value?: Token['value'], children?: Token['children'] }): Token { - return new Token(kind, pos, opts?.hasLeftSpacing, opts?.value, opts?.children); +export function expectTokenKind(token: Token, kind: T): asserts token is Token & { kind: T } { + if (token.kind !== kind) { + throw unexpectedTokenError(token.kind, token.pos); + } } diff --git a/src/parser/utils.ts b/src/parser/utils.ts index c3e97d77..753e4fbf 100644 --- a/src/parser/utils.ts +++ b/src/parser/utils.ts @@ -9,15 +9,11 @@ export function NODE( start: Ast.Pos, end: Ast.Pos, ): Extract { - const node: Record = { type }; - for (const key of Object.keys(params)) { - type Key = keyof typeof params; - if (params[key as Key] !== undefined) { - node[key] = params[key as Key]; - } - } - node.loc = { start, end }; - return node as Extract; + return { type, ...params, loc: { start, end } } as Extract; +} + +export function LOC(start: Ast.Pos, end: Ast.Pos): Ast.Loc { + return { start, end }; } export function CALL_NODE( diff --git a/src/parser/visit.ts b/src/parser/visit.ts index a5686997..e4cade6d 100644 --- a/src/parser/visit.ts +++ b/src/parser/visit.ts @@ -1,10 +1,10 @@ import type * as Ast from '../node.js'; -export function visitNode(node: Ast.Node, fn: (node: Ast.Node, ancestors: Ast.Node[]) => Ast.Node): Ast.Node { +export function visitNode(node: T, fn: (node: U, ancestors: Ast.Node[]) => U): T { return visitNodeInner(node, fn, []); } -function visitNodeInner(node: Ast.Node, fn: (node: Ast.Node, ancestors: Ast.Node[]) => Ast.Node, ancestors: Ast.Node[]): Ast.Node { +function visitNodeInner(node: T, fn: (node: T, ancestors: Ast.Node[]) => T, ancestors: Ast.Node[]): T { const result = fn(node, ancestors); ancestors.push(node); @@ -12,149 +12,146 @@ function visitNodeInner(node: Ast.Node, fn: (node: Ast.Node, ancestors: Ast.Node switch (result.type) { case 'def': { if (result.varType != null) { - result.varType = visitNodeInner(result.varType, fn, ancestors) as Ast.Definition['varType']; + result.varType = visitNodeInner(result.varType, fn, ancestors); } - result.attr = result.attr.map((attr) => visitNodeInner(attr, fn, ancestors) as Ast.Attribute); - result.expr = visitNodeInner(result.expr, fn, ancestors) as Ast.Definition['expr']; + result.attr = result.attr.map((attr) => visitNodeInner(attr, fn, ancestors)); + result.expr = visitNodeInner(result.expr, fn, ancestors); break; } case 'return': { - result.expr = visitNodeInner(result.expr, fn, ancestors) as Ast.Return['expr']; + result.expr = visitNodeInner(result.expr, fn, ancestors); break; } case 'each': { - result.items = visitNodeInner(result.items, fn, ancestors) as Ast.Each['items']; - result.for = visitNodeInner(result.for, fn, ancestors) as Ast.Each['for']; + result.items = visitNodeInner(result.items, fn, ancestors); + result.for = visitNodeInner(result.for, fn, ancestors); break; } case 'for': { - if (result.from != null) { - result.from = visitNodeInner(result.from, fn, ancestors) as Ast.For['from']; + if ('from' in result) { + result.from = visitNodeInner(result.from, fn, ancestors); + result.to = visitNodeInner(result.to, fn, ancestors); + } else { + result.times = visitNodeInner(result.times, fn, ancestors); } - if (result.to != null) { - result.to = visitNodeInner(result.to, fn, ancestors) as Ast.For['to']; - } - if (result.times != null) { - result.times = visitNodeInner(result.times, fn, ancestors) as Ast.For['times']; - } - result.for = visitNodeInner(result.for, fn, ancestors) as Ast.For['for']; + result.for = visitNodeInner(result.for, fn, ancestors); break; } case 'loop': { for (let i = 0; i < result.statements.length; i++) { - result.statements[i] = visitNodeInner(result.statements[i]!, fn, ancestors) as Ast.Loop['statements'][number]; + result.statements[i] = visitNodeInner(result.statements[i]!, fn, ancestors); } break; } case 'addAssign': case 'subAssign': case 'assign': { - result.expr = visitNodeInner(result.expr, fn, ancestors) as Ast.Assign['expr']; - result.dest = visitNodeInner(result.dest, fn, ancestors) as Ast.Assign['dest']; + result.expr = visitNodeInner(result.expr, fn, ancestors); + result.dest = visitNodeInner(result.dest, fn, ancestors); break; } case 'plus': { - result.expr = visitNodeInner(result.expr, fn, ancestors) as Ast.Plus['expr']; + result.expr = visitNodeInner(result.expr, fn, ancestors); break; } case 'minus': { - result.expr = visitNodeInner(result.expr, fn, ancestors) as Ast.Minus['expr']; + result.expr = visitNodeInner(result.expr, fn, ancestors); break; } case 'not': { - result.expr = visitNodeInner(result.expr, fn, ancestors) as Ast.Not['expr']; + result.expr = visitNodeInner(result.expr, fn, ancestors); break; } case 'if': { - result.cond = visitNodeInner(result.cond, fn, ancestors) as Ast.If['cond']; - result.then = visitNodeInner(result.then, fn, ancestors) as Ast.If['then']; + result.cond = visitNodeInner(result.cond, fn, ancestors); + result.then = visitNodeInner(result.then, fn, ancestors); for (const prop of result.elseif) { - prop.cond = visitNodeInner(prop.cond, fn, ancestors) as Ast.If['elseif'][number]['cond']; - prop.then = visitNodeInner(prop.then, fn, ancestors) as Ast.If['elseif'][number]['then']; + prop.cond = visitNodeInner(prop.cond, fn, ancestors); + prop.then = visitNodeInner(prop.then, fn, ancestors); } if (result.else != null) { - result.else = visitNodeInner(result.else, fn, ancestors) as Ast.If['else']; + result.else = visitNodeInner(result.else, fn, ancestors); } break; } case 'fn': { for (const param of result.params) { if (param.default) { - param.default = visitNodeInner(param.default!, fn, ancestors) as Ast.Fn['params'][number]['default']; + param.default = visitNodeInner(param.default!, fn, ancestors); } if (param.argType != null) { - param.argType = visitNodeInner(param.argType, fn, ancestors) as Ast.Fn['params'][number]['argType']; + param.argType = visitNodeInner(param.argType, fn, ancestors); } } if (result.retType != null) { - result.retType = visitNodeInner(result.retType, fn, ancestors) as Ast.Fn['retType']; + result.retType = visitNodeInner(result.retType, fn, ancestors); } for (let i = 0; i < result.children.length; i++) { - result.children[i] = visitNodeInner(result.children[i]!, fn, ancestors) as Ast.Fn['children'][number]; + result.children[i] = visitNodeInner(result.children[i]!, fn, ancestors); } break; } case 'match': { - result.about = visitNodeInner(result.about, fn, ancestors) as Ast.Match['about']; + result.about = visitNodeInner(result.about, fn, ancestors); for (const prop of result.qs) { - prop.q = visitNodeInner(prop.q, fn, ancestors) as Ast.Match['qs'][number]['q']; - prop.a = visitNodeInner(prop.a, fn, ancestors) as Ast.Match['qs'][number]['a']; + prop.q = visitNodeInner(prop.q, fn, ancestors); + prop.a = visitNodeInner(prop.a, fn, ancestors); } if (result.default != null) { - result.default = visitNodeInner(result.default, fn, ancestors) as Ast.Match['default']; + result.default = visitNodeInner(result.default, fn, ancestors); } break; } case 'block': { for (let i = 0; i < result.statements.length; i++) { - result.statements[i] = visitNodeInner(result.statements[i]!, fn, ancestors) as Ast.Block['statements'][number]; + result.statements[i] = visitNodeInner(result.statements[i]!, fn, ancestors); } break; } case 'exists': { - result.identifier = visitNodeInner(result.identifier, fn, ancestors) as Ast.Exists['identifier']; + result.identifier = visitNodeInner(result.identifier, fn, ancestors); break; } case 'tmpl': { for (let i = 0; i < result.tmpl.length; i++) { const item = result.tmpl[i]!; if (typeof item !== 'string') { - result.tmpl[i] = visitNodeInner(item, fn, ancestors) as Ast.Tmpl['tmpl'][number]; + result.tmpl[i] = visitNodeInner(item, fn, ancestors); } } break; } case 'obj': { for (const item of result.value) { - result.value.set(item[0], visitNodeInner(item[1], fn, ancestors) as Ast.Expression); + result.value.set(item[0], visitNodeInner(item[1], fn, ancestors)); } break; } case 'arr': { for (let i = 0; i < result.value.length; i++) { - result.value[i] = visitNodeInner(result.value[i]!, fn, ancestors) as Ast.Arr['value'][number]; + result.value[i] = visitNodeInner(result.value[i]!, fn, ancestors); } break; } case 'call': { - result.target = visitNodeInner(result.target, fn, ancestors) as Ast.Call['target']; + result.target = visitNodeInner(result.target, fn, ancestors); for (let i = 0; i < result.args.length; i++) { - result.args[i] = visitNodeInner(result.args[i]!, fn, ancestors) as Ast.Call['args'][number]; + result.args[i] = visitNodeInner(result.args[i]!, fn, ancestors); } break; } case 'index': { - result.target = visitNodeInner(result.target, fn, ancestors) as Ast.Index['target']; - result.index = visitNodeInner(result.index, fn, ancestors) as Ast.Index['index']; + result.target = visitNodeInner(result.target, fn, ancestors); + result.index = visitNodeInner(result.index, fn, ancestors); break; } case 'prop': { - result.target = visitNodeInner(result.target, fn, ancestors) as Ast.Prop['target']; + result.target = visitNodeInner(result.target, fn, ancestors); break; } case 'ns': { for (let i = 0; i < result.members.length; i++) { - result.members[i] = visitNodeInner(result.members[i]!, fn, ancestors) as (typeof result.members)[number]; + result.members[i] = visitNodeInner(result.members[i]!, fn, ancestors); } break; } @@ -173,50 +170,20 @@ function visitNodeInner(node: Ast.Node, fn: (node: Ast.Node, ancestors: Ast.Node case 'neq': case 'and': case 'or': { - result.left = visitNodeInner(result.left, fn, ancestors) as ( - Ast.Pow | - Ast.Mul | - Ast.Div | - Ast.Rem | - Ast.Add | - Ast.Sub | - Ast.Lt | - Ast.Lteq | - Ast.Gt | - Ast.Gteq | - Ast.Eq | - Ast.Neq | - Ast.And | - Ast.Or - )['left']; - result.right = visitNodeInner(result.right, fn, ancestors) as ( - Ast.Pow | - Ast.Mul | - Ast.Div | - Ast.Rem | - Ast.Add | - Ast.Sub | - Ast.Lt | - Ast.Lteq | - Ast.Gt | - Ast.Gteq | - Ast.Eq | - Ast.Neq | - Ast.And | - Ast.Or - )['right']; + result.left = visitNodeInner(result.left, fn, ancestors); + result.right = visitNodeInner(result.right, fn, ancestors); break; } case 'fnTypeSource': { for (let i = 0; i < result.params.length; i++) { - result.params[i] = visitNodeInner(result.params[i]!, fn, ancestors) as Ast.FnTypeSource['params'][number]; + result.params[i] = visitNodeInner(result.params[i]!, fn, ancestors); } break; } case 'unionTypeSource': { for (let i = 0; i < result.inners.length; i++) { - result.inners[i] = visitNodeInner(result.inners[i]!, fn, ancestors) as Ast.UnionTypeSource['inners'][number]; + result.inners[i] = visitNodeInner(result.inners[i]!, fn, ancestors); } break; } diff --git a/test/parser.ts b/test/parser.ts index c2588b01..fa386fa0 100644 --- a/test/parser.ts +++ b/test/parser.ts @@ -8,14 +8,14 @@ describe('CharStream', () => { test.concurrent('char', async () => { const source = 'abc'; const stream = new CharStream(source); - assert.strictEqual('a', stream.char); + assert.strictEqual('a', stream.char()); }); test.concurrent('next', async () => { const source = 'abc'; const stream = new CharStream(source); stream.next(); - assert.strictEqual('b', stream.char); + assert.strictEqual('b', stream.char()); }); describe('prev', () => { @@ -23,37 +23,37 @@ describe('CharStream', () => { const source = 'abc'; const stream = new CharStream(source); stream.next(); - assert.strictEqual('b', stream.char); + assert.strictEqual('b', stream.char()); stream.prev(); - assert.strictEqual('a', stream.char); + assert.strictEqual('a', stream.char()); }); test.concurrent('境界外には移動しない', async () => { const source = 'abc'; const stream = new CharStream(source); stream.prev(); - assert.strictEqual('a', stream.char); + assert.strictEqual('a', stream.char()); }); }); test.concurrent('eof', async () => { const source = 'abc'; const stream = new CharStream(source); - assert.strictEqual(false, stream.eof); + assert.strictEqual(false, stream.eof()); stream.next(); - assert.strictEqual(false, stream.eof); + assert.strictEqual(false, stream.eof()); stream.next(); - assert.strictEqual(false, stream.eof); + assert.strictEqual(false, stream.eof()); stream.next(); - assert.strictEqual(true, stream.eof); + assert.strictEqual(true, stream.eof()); }); test.concurrent('EOFでcharを参照するとエラー', async () => { const source = ''; const stream = new CharStream(source); - assert.strictEqual(true, stream.eof); + assert.strictEqual(true, stream.eof()); try { - stream.char; + stream.char(); } catch (e) { return; } @@ -63,13 +63,13 @@ describe('CharStream', () => { test.concurrent('CRは読み飛ばされる', async () => { const source = 'a\r\nb'; const stream = new CharStream(source); - assert.strictEqual('a', stream.char); + assert.strictEqual('a', stream.char()); stream.next(); - assert.strictEqual('\n', stream.char); + assert.strictEqual('\n', stream.char()); stream.next(); - assert.strictEqual('b', stream.char); + assert.strictEqual('b', stream.char()); stream.next(); - assert.strictEqual(true, stream.eof); + assert.strictEqual(true, stream.eof()); }); }); @@ -79,7 +79,10 @@ describe('Scanner', () => { return stream; } function next(stream: Scanner, kind: TokenKind, pos: TokenPosition, opts: { hasLeftSpacing?: boolean, value?: string }) { - assert.deepStrictEqual(stream.getToken(), TOKEN(kind, pos, opts)); + if (opts.hasLeftSpacing == null) { + opts.hasLeftSpacing = false; + } + assert.deepStrictEqual(stream.getToken(), TOKEN(kind as any, pos, opts as any)); stream.next(); } @@ -139,7 +142,7 @@ describe('Scanner', () => { test.concurrent('lookahead', async () => { const source = '@abc() { }'; const stream = init(source); - assert.deepStrictEqual(stream.lookahead(1), TOKEN(TokenKind.Identifier, { line: 1, column: 2 }, { value: 'abc' })); + assert.deepStrictEqual(stream.lookahead(1), TOKEN(TokenKind.Identifier, { line: 1, column: 2 }, { hasLeftSpacing: false, value: 'abc' })); next(stream, TokenKind.At, { line: 1, column: 1 }, { }); next(stream, TokenKind.Identifier, { line: 1, column: 2 }, { value: 'abc' }); next(stream, TokenKind.OpenParen, { line: 1, column: 5 }, { }); diff --git a/unreleased/ast-node-for-type.md b/unreleased/ast-node-for-type.md new file mode 100644 index 00000000..f89a69b0 --- /dev/null +++ b/unreleased/ast-node-for-type.md @@ -0,0 +1 @@ +- **Breaking Change** For Hosts: Ast.Forの型がより厳格になりました。