progress in automated scaling

utils/scaling_scripts/parse_elpa1

 #! /bin/bash
-echo nodes total tridiag solve trans_ev
+echo nodes total tridiag solve trans_ev file
 for f in *.txt 
 do
     #echo "processing $f... "
@@ -14,5 +14,6 @@ do
     S+=" "`grep "|_ tridiag_" $f | awk '{print $3}'`
     S+=" "`grep "|_ solve  " $f | awk '{print $3}'`
     S+=" "`grep "|_ trans_ev" $f | awk '{print $3}'`
+    S+=" "`echo $f`
     echo $S
 done

utils/scaling_scripts/parse_elpa2

 #! /bin/bash
-echo nodes total bandred tridiag solve trans_ev_to_band trans_ev_to_full
+echo nodes total bandred tridiag solve trans_ev_to_band trans_ev_to_full file
 for f in *.txt 
 do
     #echo "processing $f... "
@@ -16,6 +16,7 @@ do
     S+=" "`grep "|_ solve " $f | awk '{print $3}'`
     S+=" "`grep "|_ trans_ev_to_band " $f | awk '{print $3}'`
     S+=" "`grep "|_ trans_ev_to_full " $f | awk '{print $3}'`
+    S+=" "`echo $f`
 
     echo $S
 done

utils/scaling_scripts/parse_mkl

 #! /bin/bash
-echo nodes total
+echo nodes total file
 for f in *.txt 
 do
     #echo "processing $f... "
@@ -10,5 +10,6 @@ do
         continue
     fi
     S+=" "$TOTAL 
+    S+=" "`echo $f`
     echo $S
 done

utils/scaling_scripts/process.py

@@ -2,25 +2,35 @@
 import os
 import subprocess
 
-rootdir = "results"
-path = subprocess.check_output('pwd')[:-1] + "/"
+directories = ["out", "out_1", "out_1818"]
+with open("results_raw.txt", "w") as logfile:
+  for rootdir in directories:
+    path = subprocess.check_output('pwd')[:-1].decode() + "/"
+    for subdir, dirs, files in os.walk(rootdir):
+      #  for file in files:
+      #      print os.path.join(subdir, file)
+       # print subdir, dirs, files 
+        if(len(files) != 0):
+            #print subdir, dirs, files 
+            print("cd " + path + subdir)
+            logfile.write(subdir + "\n")
+            os.chdir(path + subdir)
 
-for subdir, dirs, files in os.walk(rootdir):
-  #  for file in files:
-  #      print os.path.join(subdir, file)
-   # print subdir, dirs, files 
-    if(len(files) != 0):
-        #print subdir, dirs, files 
-        #print("cd " + path + subdir)
-        os.chdir(path + subdir)
+            method = subdir.split("/")[-1]
+            with open("tab.txt", "w") as outfile:
+              if(method == "elpa1"):
+                parser = "parse_elpa1"
+              elif(method == "elpa2"):
+                parser = "parse_elpa2"
+              else:
+                parser = "parse_mkl"
 
-        method = subdir.split("/")[-1]
-        with open("tab.txt", "w") as outfile:
-            if(method == "elpa1"):
-                subprocess.call("parse_elpa1", stdout=outfile) 
-            elif(method == "elpa2"):
-                subprocess.call("parse_elpa2", stdout=outfile)
-            else:
-                subprocess.call("parse_mkl", stdout=outfile)
+              ps = subprocess.Popen(path + parser, stdout=subprocess.PIPE)
+              output = subprocess.check_output(('column', '-t'), stdin = ps.stdout).decode()
+              ps.wait()
+              outfile.write(output)
+              logfile.write(output)
+              logfile.write("\n")
 
-        os.chdir(path)
+
+            os.chdir(path)

utils/scaling_scripts/process_raw.py

+#! /usr/bin/env python
+from collections import namedtuple
+from subprocess import Popen, PIPE, STDOUT
+
+EntryType = namedtuple("EntryType", "directory run dir_raw header lines")
+with open("results_raw.txt", "r") as rawfile:
+  expect = "directory"
+  entries = []
+  entry = []
+  for line in rawfile:
+    if(line == "\n"):
+      if(expect == "entry"):
+        entries.append(entry)
+        expect = "directory"
+      else:
+        raise("cannot be")
+    else:
+      if(expect == "directory"):
+        directory = line.split('/', 1)[0]
+        run = line.split('/',1)[1][:-1].split('/')
+        run[2] = int(run[2])
+        run[3] = int(run[3])
+        dir_raw = line[:-1]
+        entry = EntryType(directory, run , dir_raw, [], [])
+        expect = "heading"
+        #print("next entry: " + line, end="")
+      elif (expect == "heading"):
+        entry.header.extend(line.split())
+        expect = "entry"
+        #print("heading: " + line, end="")
+      elif (expect == "entry"):
+        entry.lines.append(line.split())
+
+
+entries.sort(key = lambda entry: entry.run)
+entries.sort(key = lambda entry: entry.run[0], reverse=True)
+print(entries)
+for e in entries:
+  print(e.directory, e.run)
+
+with open("results_sorted.txt", "w") as sortedfile:
+  lines_to_write = []
+  for idx in range(len(entries)):
+    e = entries[idx]
+    for l in e.lines:
+      lw = l[:-1]
+      lw.append(e.dir_raw + "/" + l[-1])
+      lines_to_write.append(lw)
+
+    if(idx == len(entries) - 1 or e.run != entries[idx+1].run):
+      if(e.run != entries[0].run):
+        sortedfile.write("\n")
+      sortedfile.write(" ".join(str(x) for x in e.run) + "\n")
+      group_string = " ".join(str(x) for x in e.header) + "\n"
+      lines_to_write.sort(key = lambda line: int(line[0]))
+      for lw in lines_to_write:
+        group_string += " ".join(lw) + "\n"
+
+      p = Popen(['column', '-t'], stdout=PIPE, stdin=PIPE, stderr=PIPE)
+      stdout_data = p.communicate(input=group_string.encode())[0]
+      sortedfile.write(stdout_data.decode())
+
+      lines_to_write = []
+
+