Skip to content

Instantly share code, notes, and snippets.

@canimus
Created October 28, 2019 22:39
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save canimus/39e96798c167c469005eb9f48d39de27 to your computer and use it in GitHub Desktop.
Save canimus/39e96798c167c469005eb9f48d39de27 to your computer and use it in GitHub Desktop.
XML Tag Counter with Python Multiprocess
# Author: Herminio Vazquez
# Date: 2019-10-28
# Project: DataStore
# @canimus
import os
from multiprocessing import Pool
import multiprocessing
import subprocess
from functional import seq
files = seq(os.listdir(".")).filter(lambda x: x.startswith("zplit"))
tags = open("tags.log").readlines()
numbers = [1,2,3,4,5,6,7]
def c(x):
name = multiprocessing.current_process().name
pos = int(name.split("-")[-1])-1
with open(files[pos]) as infile:
for file_name in infile.readlines():
total = []
for t in tags:
command = ['grep', '-c', f'<{t.strip()}[ >:/]', file_name.strip()]
proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = proc.communicate()
total.append(out.strip().decode("utf-8"))
print(" ".join(total))
if __name__ == '__main__':
p = Pool(8)
try:
p.map(c, files)
except Exception as e:
print(e)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment