python实现词法分析器


Posted in Python onJanuary 31, 2019

简单Python词法分析器实现,供大家参考,具体内容如下

词法分析器状态转换图:

python实现词法分析器

词法分析器总流程图:

python实现词法分析器

预处理程序:

python实现词法分析器

词法分析器:

python实现词法分析器

词法分析器程序详细设计

python实现词法分析器

详细代码实现:

#!/usr/bin/env python3.4
# coding=utf-8
import sys
import string


keywards = {}

# 关键字部分
keywards['False'] = 101
keywards['class'] = 102
keywards['finally'] = 103
keywards['is'] = 104
keywards['return'] = 105
keywards['None'] = 106
keywards['continue'] = 107
keywards['for'] = 108
keywards['lambda'] = 109
keywards['try'] = 110
keywards['True'] = 111
keywards['def'] = 112
keywards['from'] = 113
keywards['nonlocal'] = 114
keywards['while'] = 115
keywards['and'] = 116
keywards['del'] = 117
keywards['global'] = 118
keywards['not'] = 119
keywards['with'] = 120
keywards['as'] = 121
keywards['elif'] = 122
keywards['if'] = 123
keywards['or'] = 124
keywards['yield'] = 125
keywards['assert'] = 126
keywards['else'] = 127
keywards['import'] = 128
keywards['pass'] = 129
keywards['break'] = 130
keywards['except'] = 131
keywards['in'] = 132
keywards['raise'] = 133

# 符号
keywards['+'] = 201
keywards['-'] = 202
keywards['*'] = 203
keywards['/'] = 204
keywards['='] = 205
keywards[':'] = 206
keywards['<'] = 207
keywards['>'] = 208
keywards['%'] = 209
keywards['&'] = 210
keywards['!'] = 211
keywards['('] = 212
keywards[')'] = 213
keywards['['] = 214
keywards[']'] = 215
keywards['{'] = 216
keywards['}'] = 217
keywards['#'] = 218
keywards['|'] = 219
keywards[','] = 220
# 变量
# keywards['var'] = 301

# 常量
# keywards['const'] = 401

# Error
# keywards['const'] = 501

signlist = {}


# 预处理函数,将文件中的空格,换行等无关字符处理掉
def pretreatment(file_name):
 try:
  fp_read = open(file_name, 'r')
  fp_write = open('file.tmp', 'w')
  sign = 0
  while True:
   read = fp_read.readline()
   if not read:
    break
   length = len(read)
   i = -1
   while i < length - 1:
    i += 1
    if sign == 0:
     if read[i] == ' ':
      continue
    if read[i] == '#':
     break
    elif read[i] == ' ':
     if sign == 1:
      continue
     else:
      sign = 1
      fp_write.write(' ')
    elif read[i] == '\t':
     if sign == 1:
      continue
     else:
      sign = 1
      fp_write.write(' ')
    elif read[i] == '\n':
     if sign == 1:
      continue
     else:
      fp_write.write(' ')
      sign = 1
    elif read[i] == '"':
     fp_write.write(read[i])
     i += 1
     while i < length and read[i] != '"':
      fp_write.write(read[i])
      i += 1
     if i >= length:
      break
     fp_write.write(read[i])
    elif read[i] == "'":
     fp_write.write(read[i])
     i += 1
     while i < length and read[i] != "'":
      fp_write.write(read[i])
      i += 1
     if i >= length:
      break
     fp_write.write(read[i])
    else:
     sign = 3
     fp_write.write(read[i])
 except Exception:
  print(file_name, ': This FileName Not Found!')


def save(string):
 if string in keywards.keys():
  if string not in signlist.keys():
   signlist[string] = keywards[string]
 else:
  try:
   float(string)
   save_const(string)
  except ValueError:
   save_var(string)


def save_var(string):
 if string not in signlist.keys():
  if len(string.strip()) < 1:
   pass
  else:
   if is_signal(string) == 1:
    signlist[string] = 301
   else:
    signlist[string] = 501


def save_const(string):
 if string not in signlist.keys():
  signlist[string] = 401


def save_error(string):
 if string not in signlist.keys():
  signlist[string] = 501


def is_signal(s):
 if s[0] == '_' or s[0] in string.ascii_letters:
  for i in s:
   if i in string.ascii_letters or i == '_' or i in string.digits:
    pass
   else:
    return 0
  return 1
 else:
  return 0


def recognition(filename):
 try:
  fp_read = open(filename, 'r')
  string = ""
  sign = 0
  while True:
   read = fp_read.read(1)
   if not read:
    break

   if read == ' ':
    if len(string.strip()) < 1:
     sign = 0
     pass
    else:
     if sign == 1 or sign == 2:
      string += read
     else:
      save(string)
      string = ""
      sign = 0
   elif read == '(':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('(')
   elif read == ')':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(')')
   elif read == '[':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('[')
   elif read == ']':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(']')
   elif read == '{':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('{')
   elif read == '}':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save('}')
   elif read == '<':
    save(string)
    string = ""
    save('<')
   elif read == '>':
    save(string)
    string = ""
    save('>')
   elif read == ',':
    save(string)
    string = ""
    save(',')
   elif read == "'":
    string += read
    if sign == 1:
     sign = 0
     save_const(string)
     string = ""
    else:
     if sign != 2:
      sign = 1
   elif read == '"':
    string += read
    if sign == 2:
     sign = 0
     save_const(string)
     string = ""
    else:
     if sign != 1:
      sign = 2
   elif read == ':':
    if sign == 1 or sign == 2:
     string += read
    else:
     save(string)
     string = ""
     save(':')
   elif read == '+':
    save(string)
    string = ""
    save('+')
   elif read == '=':
    save(string)
    string = ""
    save('=')
   else:
    string += read

 except Exception as e:
  print(e)


def main():
 if len(sys.argv) < 2:
  print("Please Input FileName")
 else:
  pretreatment(sys.argv[1])
 recognition('file.tmp')
 for i in signlist.keys():
  print("(", signlist[i], ",", i, ")")


if __name__ == '__main__':
 main()

以上就是本文的全部内容,希望对大家的学习有所帮助,也希望大家多多支持三水点靠木。

Python 相关文章推荐
Python使用CMD模块更优雅的运行脚本
May 11 Python
python使用PyGame模块播放声音的方法
May 20 Python
Python 实现文件的全备份和差异备份详解
Dec 27 Python
关于numpy中np.nonzero()函数用法的详解
Feb 07 Python
Python补齐字符串长度的实例
Nov 15 Python
python反编译学习之字节码详解
May 19 Python
Python3多目标赋值及共享引用注意事项
May 27 Python
Python变量访问权限控制详解
Jun 29 Python
django echarts饼图数据动态加载的实例
Aug 12 Python
Python的垃圾回收机制详解
Aug 28 Python
Windows下pycharm创建Django 项目(虚拟环境)过程解析
Sep 16 Python
Python坐标线性插值应用实现
Nov 13 Python
使用Python 统计高频字数的方法
Jan 31 #Python
python3正则提取字符串里的中文实例
Jan 31 #Python
python统计中文字符数量的两种方法
Jan 31 #Python
解决python写入带有中文的字符到文件错误的问题
Jan 31 #Python
在Python中居然可以定义两个同名通参数的函数
Jan 31 #Python
Python的条件表达式和lambda表达式实例
Jan 31 #Python
OpenCV+Python识别车牌和字符分割的实现
Jan 31 #Python
You might like
hessian 在PHP中的使用介绍
2010/12/13 PHP
PHP基于ip2long实现IP转换整形
2020/12/11 PHP
IE中jscript/javascript的条件编译
2006/09/07 Javascript
初学js 新节点的创建 删除 的步骤
2011/07/04 Javascript
select标记美化--JS式插件、后期加载
2013/04/01 Javascript
nodejs的10个性能优化技巧
2014/07/15 NodeJs
使用Javascript简单实现图片无缝滚动
2014/12/05 Javascript
浅谈JavaScript的Polymer框架中的behaviors对象
2015/07/29 Javascript
jQuery实现点击小图片淡入淡出显示大图片特效
2015/09/09 Javascript
jQuery实现下拉加载功能实例代码
2016/04/01 Javascript
详解AngularJS脏检查机制及$timeout的妙用
2017/06/19 Javascript
vue iView 上传组件之手动上传功能
2018/03/16 Javascript
值得收藏的八个常用的js正则表达式
2018/10/19 Javascript
koa源码中promise的解读
2018/11/13 Javascript
13 个npm 快速开发技巧(推荐)
2019/07/04 Javascript
如何用vue-cli3脚手架搭建一个基于ts的基础脚手架的方法
2019/12/12 Javascript
原生js实现无缝轮播图
2020/01/11 Javascript
JS异步宏队列与微队列原理区别详解
2020/07/02 Javascript
python图片验证码生成代码
2016/07/02 Python
Python中的 enum 模块源码详析
2019/01/09 Python
pandas 使用均值填充缺失值列的小技巧分享
2019/07/04 Python
Pytorch 使用CNN图像分类的实现
2020/06/16 Python
CSS3实现div从下往上滑入滑出效果示例
2020/04/28 HTML / CSS
印度领先的眼镜电子商务网站:Lenskart
2019/12/16 全球购物
Web Service面试题:如何搭建Axis2的开发环境
2012/06/20 面试题
文员自我评价怎么写
2013/09/19 职场文书
社区法制宣传月活动总结
2015/05/07 职场文书
孙振耀退休感言
2015/08/01 职场文书
初中英语教学随笔
2015/08/15 职场文书
2015团员个人年度总结
2015/11/24 职场文书
建立共青团委员会的请示
2019/04/02 职场文书
详解 TypeScript 枚举类型
2021/11/02 Javascript
DIV CSS实现网页背景半透明效果
2021/12/06 HTML / CSS
Python借助with语句实现代码段只执行有限次
2022/03/23 Python
Redis实现短信验证码登录的示例代码
2022/06/14 Redis
解决Git推送错误non-fast-forward的方法
2022/06/25 Servers