python 正则表达式过滤文本中的html标签源代码解析

#py2.7#coding:utf-8import reimport osimport chardetdef filter_tag(htmlstr):re_cdata = re.compile(']*>', re.I)re_script = re.compile(']*>[^', re.I) #过滤脚本re_style = re.compile(']*>

爱知菜

5600人浏览 · 2013-10-20 20:03:31

爱知菜 · 2013-10-20 20:03:31 发布

#py2.7
#coding:utf-8

import re
import os
import chardet

def filter_tag(htmlstr):
    re_cdata = re.compile('<!DOCTYPE HTML PUBLIC[^>]*>', re.I)
    re_script = re.compile('<\s*script[^>]*>[^<]*<\s*/\s*script\s*>', re.I) #过滤脚本
    re_style = re.compile('<\s*style[^>]*>[^<]*<\s*/\s*style\s*>', re.I) #过滤style
    re_br = re.compile('<br\s*?/?>')
    re_h = re.compile('</?\w+[^>]*>')
    re_comment = re.compile('<!--[\s\S]*-->')
    s = re_cdata.sub('', htmlstr)
    s = re_script.sub('', s)
    s=re_style.sub('',s)
    s=re_br.sub('\n',s)
    s=re_h.sub(' ',s)
    s=re_comment.sub('',s)
    blank_line=re.compile('\n+')
    s=blank_line.sub('\n',s)
    s=re.sub('\s+',' ',s)
    s=replaceCharEntity(s)
    return s

def replaceCharEntity(htmlstr):
    CHAR_ENTITIES={'nbsp':'','160':'',
                    'lt':'<','60':'<',
                    'gt':'>','62':'>',
                    'amp':'&','38':'&',
                    'quot':'"','34':'"'}
    re_charEntity=re.compile(r'&#?(?P<name>\w+);') #命名组,把 匹配字段中\w+的部分命名为name,可以用group函数获取
    sz=re_charEntity.search(htmlstr)
    while sz:
        #entity=sz.group()
        key=sz.group('name') #命名组的获取
        try:
            htmlstr=re_charEntity.sub(CHAR_ENTITIES[key],htmlstr,1) #1表示替换第一个匹配
            sz=re_charEntity.search(htmlstr)
        except KeyError:
            htmlstr=re_charEntity.sub('',htmlstr,1)
            sz=re_charEntity.search(htmlstr)
    return htmlstr

if __name__=='__main__':
    cpath=os.getcwd()
    
    for root,dirs,files in os.walk(cpath):
        for file in files:
            if file.endswith('htm') or file.endswith('html'):
                f=open(root+os.path.sep+file)
                stream=f.read()
                htmlstr =stream.decode(chardet.detect(stream)['encoding'])
                rs=filter_tag(htmlstr)
                f.close()
                txtname=re.sub(r'.htm*$','.txt',file)
                print txtname
                f=open(root+os.path.sep+txtname,'w')
                f.write(rs.encode('utf-8'))
                f.close()