自制编程语言（第三弹）定义Token

终于到了激动人心的实现时候了。为了实现我们的自制语言，我们需要的步骤为：

1. 词法分析
2. 语法分析
3. 语义分析（此处不设置）
4. 解释器

详细完整的代码可以点击这里查看github项目。

词法分析：

将代码片段识别为关键词、标识符、操作符、数字字面量等Token。

首先我们定义需要定义我们语言中所用到所有token类型

public enum TokenType
{
   
    NAME,
    INT_CONST,
    REAL_CONST,
    STR,
    NEWLINE,
    INDENT,
    DEDENT,
    COLON,
    LPAREN,
    RPAREN,
    LBRACK,
    RBRACK,
    ASSIGN,
    COMMENT,
    DOT,
    IF,
    ELSE,
    ELIF,
    CASE,
    WHEN,
    EQ,
    NE,
    LT,
    LE,
    GT,
    GE,
    PLUS,
    MINUS,
    MUL,
    DIV,
    FLOORDIV,
    MOD,
    POW,
    NOT,
    AND,
    OR,
    TRUE,
    FALSE,
    NONE,
    EOF
}

定义一个Token类，用于封装和包含Token信息

public class Token

public TokenType Type {
    get; set; }
public object Value {
    get; set; }
public int Line {
    get; set; }
public int Column {
    get; set; }

对保留字做个映射

public static Dictionary<string, TokenType> ReservedKeywords {
    get; } = new Dictionary<string, TokenType>()
{
   
    {
   "的",TokenType.DOT },
    {
   "如果",TokenType.IF},
    {
   "否则",TokenType.ELSE},
    {
   "否则如果",TokenType.ELIF},
    {
   "选择",TokenType.CASE},
    {
   "选项",TokenType.WHEN},
    {
   "非",TokenType.NOT},
    {
   "并且",TokenType.AND},
    {
   "或者",TokenType.OR},
    {
   "幂",TokenType.POW},
    {
   "true",TokenType.TRUE},
    {
   "false",TokenType.FALSE},
    {
   "none",TokenType.NONE},
    {
   "真",TokenType.TRUE},
    {
   "假",TokenType.FALSE},
    {
   "开",TokenType.TRUE},
    {
   "关",TokenType.FALSE},
    {
   "空",TokenType.NONE},
};
public static Token GetReservedKeywords(string value, int line, int column)
{
   
    if (ReservedKeywords.ContainsKey(value))
        return new Token(ReservedKeywords[value], value, line, column);
    return null;
}

设置构造函数和ToString

public Token(TokenType type, object value, int line, int column)
{
   
    Type = type;
    Value = value;
    Line = line;
    Column = column;
}
public override string ToString()
{
   
    return string.Format("Token({0}, {1}, position = {2}:{3})", Type, Value, Line, Column);
}