将 LexToken 转换为列表 Python

Question

我有一个用于 html 标记的词法分析器，它返回并打印给定 html 字符串中的 lextoken 对象

我有一个解析器，它将标记作为列表和语法作为输入，如果标记集在语法中形成有效字符串，则返回 true

我想把这些程序组合成一个完整的词法分析器程序

但问题是在第二个程序中，令牌是列表形式，第一个程序的输出是 lextoken

词法分析器

import ply.lex as lex

tokens = (
        'LANGLE',       # <
        'LANGLESLASH',  # </
        'RANGLE',       # >
        'SLASHRANGLE',  # />
        'EQUAL',        # =
        'STRING',       # "144"
        'WORD',         # 'Welcome' in "Welcome to my webpage."
        'NUMBER'        # 12, 5.6, -1., 3.14159, -8.1, 867.5309

)

t_ignore                = ' \t\v\r' # shortcut for whitespace

states = (
        ('htmlcomment', 'exclusive'),   # <!--
)

def t_htmlcomment(t):
        r'<!--'
        t.lexer.begin('htmlcomment')

def t_htmlcomment_end(t):
        r'-->'
        t.lexer.lineno += t.value.count('\n')
        t.lexer.begin('INITIAL')
        pass

def t_htmlcomment_error(t):
        t.lexer.skip(1)

def t_LANGLESLASH(t):
        r'</'
        return t

def t_LANGLE(t):
        r'<'
        return t

def t_SLASHRANGLE(t):
        r'/>'
        return t

def t_RANGLE(t):
        r'>'
        return t

def t_EQUAL(t):
        r'='
        return t

def t_STRING(t):
        r'"[^"]*"'
        t.value = t.value[1:-1] # drop "surrounding quotes"
        return t

def t_WORD(t):
        r'[^ <>]+'
        return t

webpage = "hello <!-- comment --> 123456 <b> Bushra </b> all"
htmllexer = lex.lex()
htmllexer.input(webpage)
while True:
        tok = htmllexer.token()
        if not tok: break
        print tok

这是我的解析器

work_count = 0      # track one notion of "time taken"

def addtoset(theset,index,elt):
  if not (elt in theset[index]):
    theset[index] = [elt] + theset[index]
    return True
  return False

def parse(tokens,grammar):
  global work_count
  work_count = 0
  tokens = tokens + [ "end_of_input_marker" ]
  chart = {}
  start_rule = grammar[0]
  for i in range(len(tokens)+1):
    chart[i] = [ ]
  start_state = (start_rule[0], [], start_rule[1], 0)
  chart[0] = [ start_state ]
  for i in range(len(tokens)):
    while True:
      changes = False
      for state in chart[i]:
        # State ===   x -> a b . c d , j
        x = state[0]
        ab = state[1]
        cd = state[2]
        j = state[3]


        next_states = [ (rule[0],[],rule[1],i)
          for rule in grammar if cd <> [] and cd[0] == rule[0] ]
        work_count = work_count + len(grammar)
        for next_state in next_states:
          changes = addtoset(chart,i,next_state) or changes

        if cd <> [] and tokens[i] == cd[0]:
          next_state = (x, ab + [cd[0]], cd[1:], j)
          changes = addtoset(chart,i+1,next_state) or changes

        next_states = [ (jstate[0], jstate[1] + [x], (jstate[2])[1:],
                         jstate[3] )
          for jstate in chart[j]
          if cd == [] and jstate[2] <> [] and (jstate[2])[0] == x ]
        work_count = work_count + len(chart[j])
        for next_state in next_states:
          changes = addtoset(chart,i,next_state) or changes


      # We're done if nothing changed!
      if not changes:
        break



  accepting_state = (start_rule[0], start_rule[1], [], 0)
  return accepting_state in chart[len(tokens)-1]

grammar = [ 
    ("html", ["element", "html"]),
    ("html", [ ]),
    ("element", ["word"]),
    ("element", ["tag-open","word","tag-close"]),
    ("tag-open",["<","word",">"]),
    ("tag-close",["<","/","word",">"])
    ]

tokens = [ "<", "b", ">" , "Hello", "<", "/" , "b" , ">"]
result=parse(tokens, grammar)
print result

Answer 1

你可以使用LexToken的属性value来做到这一点：

webpage = "hello <!-- comment --> 123456 <b> Bushra </b> all"
htmllexer = lex.lex()
htmllexer.input(webpage)
tokens = []
while True:
        tok = htmllexer.token()
        if not tok: break
        tokens.append(tok.value)
print tokens #['hello', '123456', '<', 'b', '>', 'Bushra', '</', 'b', '>', 'all']

所有可用的属性都可以通过dir()函数获取：

print dir(tok)