Skip to content

Commit

Permalink
refactor cmd line lexer + parser
Browse files Browse the repository at this point in the history
  • Loading branch information
@guillermooo
guillermooo committed Nov 16, 2015
1 parent 335e93b commit 305c209
Show file tree
Hide file tree
Showing 9 changed files with 428 additions and 501 deletions.
5 changes: 4 additions & 1 deletion src/cmd_line/command_node.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ export class WriteCommand implements node.CommandBase {
}

runOn(textEditor : vscode.TextEditor) : void {
if (this.args || !textEditor.document.fileName) util.showInfo("Not implemented.");
if (this.args || !textEditor.document.fileName) {
util.showInfo("Not implemented.");
return;
}
textEditor.document.save();
}
}
304 changes: 154 additions & 150 deletions src/cmd_line/lexer.ts
View file
Original file line number Diff line number Diff line change
@@ -1,171 +1,175 @@
import {State} from './lexer_state';
import * as token from './token';
import {Scanner} from './scanner';
import {Token, TokenType} from './token';

interface ScanFunction {
(state: State, tokens: token.Token[]) : ScanFunction;
// Describes a function that can lex part of a Vim command line.
interface LexFunction {
(state: Scanner, tokens: Token[]) : LexFunction;
}

export function scan(input : string) : token.Token[] {
var state = new State(input);
var tokens : token.Token[] = [];
var f : ScanFunction = scanRange; // first scanning function
export function lex(input : string) : Token[] {
// we use a character scanner as state for the lexer
var state = new Scanner(input);
var tokens : Token[] = [];
var f : LexFunction = LexFunctions.lexRange; // first lexing function
while (f) {
// Each scanning function returns the next scanning function or null.
// Each lexing function returns the next lexing function or null.
f = f(state, tokens);
}
return tokens;
}

function scanRange(state : State, tokens : token.Token[]): ScanFunction {
while (true) {
if (state.isAtEof) {
break;
}
var c = state.next();
switch (c) {
case ',':
tokens.push(new token.TokenComma());
state.ignore();
continue;
case '%':
tokens.push(new token.TokenPercent());
state.ignore();
continue;
case '$':
tokens.push(new token.TokenDollar());
state.ignore();
continue;
case '.':
tokens.push(new token.TokenDot());
state.ignore();
continue;
case '/':
return scanForwardSearch;
case '?':
return scanReverseSearch
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
return scanLineRef;
case '+':
tokens.push(new token.TokenPlus());
state.ignore();
continue;
case '-':
tokens.push(new token.TokenMinus());
state.ignore();
continue;
default:
state.backup();
return scanCommand;
}
}
return null;
function emitToken(type : TokenType, state : Scanner) : Token {
var content = state.emit();
return (content.length > 0) ? new Token(type, content) : null;
}

function scanLineRef(state : State, tokens : token.Token[]): ScanFunction {
while (true) {
if (state.isAtEof) {
var emitted = state.emit();
if (emitted) tokens.push(new token.TokenLineNumber(emitted));
return null;
}
var c = state.next();
switch (c) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
continue;
default:
state.backup();
var emitted = state.emit();
if (emitted) tokens.push(new token.TokenLineNumber(emitted));
return scanRange;
module LexFunctions {
// starts lexing a Vim command line and forwards later parts to other scanning functions.
export function lexRange(state : Scanner, tokens : Token[]): LexFunction {
while (true) {
if (state.isAtEof) {
break;
}
var c = state.next();
switch (c) {
case ',':
tokens.push(emitToken(TokenType.Comma, state));
continue;
case '%':
tokens.push(emitToken(TokenType.Percent, state));
continue;
case '$':
tokens.push(emitToken(TokenType.Dollar, state));
continue;
case '.':
tokens.push(emitToken(TokenType.Dot, state));
continue;
case '/':
return lexForwardSearch;
case '?':
return lexReverseSearch
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
return lexLineRef;
case '+':
tokens.push(emitToken(TokenType.Plus, state));
continue;
case '-':
tokens.push(emitToken(TokenType.Minus, state));
continue;
default:
state.backup();
return lexCommand;
}
}
return null;
}
return null;
}

function scanCommand(state : State, tokens : token.Token[]): ScanFunction {
state.skipWhiteSpace();
while (true) {
if (state.isAtEof) {
var emitted = state.emit();
if (emitted) tokens.push(new token.TokenCommandName(emitted));
break;
}
var c = state.next();
var lc = c.toLowerCase();
if (lc >= 'a' && lc <= 'z') {
continue;

function lexLineRef(state : Scanner, tokens : Token[]): LexFunction {
while (true) {
if (state.isAtEof) {
var emitted = emitToken(TokenType.LineNumber, state);
if (emitted) tokens.push(emitted);
return null;
}
var c = state.next();
switch (c) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
continue;
default:
state.backup();
// we're guaranteed to have a valid token here; don't check for null.
tokens.push(emitToken(TokenType.LineNumber, state));
return lexRange;
}
}
else {
state.backup();
tokens.push(new token.TokenCommandName(state.emit()));
state.skipWhiteSpace();
while (!state.isAtEof) state.next();
var args = state.emit();
if (args) tokens.push(new token.TokenCommandArgs(args));
break;
}
return null;
}
return null;
}

function scanForwardSearch(state : State, tokens : token.Token[]): ScanFunction {
state.skip('/');
var escaping : boolean;
var searchTerm = '';
while(!state.isAtEof) {
var c = state.next();
if (c == '/' && !escaping) break;
if (c == '\\') {
escaping = true;
continue;
}
else {
escaping = false;

function lexCommand(state : Scanner, tokens : Token[]): LexFunction {
state.skipWhiteSpace();
while (true) {
if (state.isAtEof) {
var emitted = emitToken(TokenType.CommandName, state);
if (emitted) tokens.push(emitted);
break;
}
var c = state.next();
var lc = c.toLowerCase();
if (lc >= 'a' && lc <= 'z') {
continue;
}
else {
state.backup();
tokens.push(emitToken(TokenType.CommandName, state));
state.skipWhiteSpace();
while (!state.isAtEof) state.next();
var args = emitToken(TokenType.CommandArgs, state);
if (args) tokens.push(args);
break;
}
}
searchTerm += c != '\\' ? c : '\\\\';
return null;
}
tokens.push(new token.TokenSlashSearch(searchTerm));
state.ignore();
if (!state.isAtEof) state.skip('/');
return scanRange;
}

function scanReverseSearch(state : State, tokens : token.Token[]): ScanFunction {
state.skip('?');
var escaping : boolean;
var searchTerm = '';
while(!state.isAtEof) {
var c = state.next();
if (c == '?' && !escaping) break;
if (c == '\\') {
escaping = true;
continue;

function lexForwardSearch(state : Scanner, tokens : Token[]): LexFunction {
state.skip('/');
var escaping : boolean;
var searchTerm = '';
while(!state.isAtEof) {
var c = state.next();
if (c == '/' && !escaping) break;
if (c == '\\') {
escaping = true;
continue;
}
else {
escaping = false;
}
searchTerm += c != '\\' ? c : '\\\\';
}
else {
escaping = false;
tokens.push(new Token(TokenType.ForwardSearch, searchTerm));
state.ignore();
if (!state.isAtEof) state.skip('/');
return lexRange;
}

function lexReverseSearch(state : Scanner, tokens : Token[]): LexFunction {
state.skip('?');
var escaping : boolean;
var searchTerm = '';
while(!state.isAtEof) {
var c = state.next();
if (c == '?' && !escaping) break;
if (c == '\\') {
escaping = true;
continue;
}
else {
escaping = false;
}
searchTerm += c != '\\' ? c : '\\\\';
}
searchTerm += c != '\\' ? c : '\\\\';
tokens.push(new Token(TokenType.ReverseSearch, searchTerm));
state.ignore();
if (!state.isAtEof) state.skip('?');
return lexRange;
}
tokens.push(new token.TokenQuestionMarkSearch(searchTerm));
state.ignore();
if (!state.isAtEof) state.skip('?');
return scanRange;
}
2 changes: 1 addition & 1 deletion src/cmd_line/main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,13 @@ function runCmdLine(s : string) : void {
}
catch (e) {
util.showInfo(e);
return;
}

if (cmd.isEmpty) {
vscode.window.showInformationMessage("empty cmdline");
}
else {
cmd.runOn(vscode.window.activeTextEditor);
// vscode.window.showInformationMessage(s);
}
}
4 changes: 2 additions & 2 deletions src/cmd_line/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,13 @@ class ParserState {
}

lex(input : string) {
this.tokens = lexer.scan(input);
this.tokens = lexer.lex(input);
}

next() : token.Token {
if (this.pos >= this.tokens.length) {
this.pos = this.tokens.length;
return new token.TokenEof();
return new token.Token(token.TokenType.Eof, '__EOF__');
}
let tok = this.tokens[this.pos];
this.pos++;
Expand Down
Loading

0 comments on commit 305c209

Please sign in to comment.