1

run.py

@@ -118,6 +118,10 @@ def clone_repo(url):
 def chr_len2(s):
     return int((len(s.encode('utf-8')) - len(s))/2 + len(s))
 
+def _md5(file):
+    with open(file,'rb') as f:
+        s = re.sub('\s+','',f.read().decode('utf8',errors='ignore'))
+        return hashlib.md5(s.encode('utf8')).hexdigest()
 
 def parse(x, y):
     s = ''
@@ -190,8 +194,7 @@ if __name__ == '__main__':
                         with open(file_path, 'r', encoding='utf8') as f:
                             content = f.read()
                         if 'GobyQuery' in content and 'ScanSteps' in content:
-                            md5 = hashlib.md5(
-                                open(file_path, 'rb').read()).hexdigest()
+                            md5 = _md5(file_path)
                             if md5 not in data:
                                 shutil.copyfile(file_path, os.path.join(
                                     root_path, 'poc', file))
@@ -207,8 +210,7 @@ if __name__ == '__main__':
     for file in os.listdir(os.path.join(root_path, 'poc')):
         if not file.endswith('.go') and not file.endswith('.json'):
             continue
-        md5 = hashlib.md5(
-            open(os.path.join(root_path, 'poc', file), 'rb').read()).hexdigest()
+        md5 = _md5(os.path.join(root_path, 'poc', file))
         md5s.append(md5)
     for md5 in [md5 for md5 in data.keys() if md5 not in md5s]:
         del data[md5]