python efficiently repeats sample code that supports GB level large files

  • 2021-01-25 07:47:17
  • OfStack

This is as follows:

import sys, re, os
def getDictList(dict):
  regx = '''[\w\~`\!\@\#\$\%\^\&\*\(\)\_\-\+\=\[\]\{\}\:\;\,\.\/\<\>\?]+'''
  with open(dict) as f:
    data =
    return re.findall(regx, data)
def rmdp(dictList):
  return list(set(dictList))
def fileSave(dictRmdp, out):
  with open(out, 'a') as f:
    for line in dictRmdp:
      f.write(line + '\n')
def main():
    dict = sys.argv[1].strip()
    out = sys.argv[2].strip()
  except Exception, e:
    print 'error:', e
    me = os.path.basename(__file__)
    print 'usage: %s <input> <output>' %me
    print 'example: %s dict.txt dict_rmdp.txt' %me
  dictList = getDictList(dict)
  dictRmdp = rmdp(dictList)
  fileSave(dictRmdp, out)
if __name__ == '__main__':

Related articles: