finished new variable grabbler

deployment-script.sh

@@ -9,10 +9,10 @@ if [ -n "`shopt globstar | grep off`" ]; then shopt -s globstar; fi
 rm ./threadr/**/README.md
 
 #run the macro handler
-echo "`find -name "*.php" -or -name "*.html" -or -name "*.css" -or -name "*.svg" | sed 's/^/python variable_grabbler.py /;s/$/ macros\/pass0\*.json/'`" | bash -
-echo "`find -name "*.php" -or -name "*.html" -or -name "*.css" -or -name "*.svg" | sed 's/^/python variable_grabbler.py /;s/$/ macros\/pass1\*.json/'`" | bash -
-echo "`find -name "*.php" -or -name "*.html" -or -name "*.css" -or -name "*.svg" | sed 's/^/python variable_grabbler.py /;s/$/ macros\/pass2\*.json/'`" | bash -
-echo "`find -name "*.php" -or -name "*.html" -or -name "*.css" -or -name "*.svg" | sed 's/^/python variable_grabbler.py /;s/$/ macros\/pass3\*.json/'`" | bash -
+echo "`find -name "*.php" -or -name "*.html" -or -name "*.css" -or -name "*.svg" | sed 's/^/python variable_grabbler.py.old /;s/$/ macros\/pass0\*.json/'`" | bash -
+echo "`find -name "*.php" -or -name "*.html" -or -name "*.css" -or -name "*.svg" | sed 's/^/python variable_grabbler.py.old /;s/$/ macros\/pass1\*.json/'`" | bash -
+echo "`find -name "*.php" -or -name "*.html" -or -name "*.css" -or -name "*.svg" | sed 's/^/python variable_grabbler.py.old /;s/$/ macros\/pass2\*.json/'`" | bash -
+echo "`find -name "*.php" -or -name "*.html" -or -name "*.css" -or -name "*.svg" | sed 's/^/python variable_grabbler.py.old /;s/$/ macros\/pass3\*.json/'`" | bash -
 
 echo "==============================================================================
 Done."

utf8test

@@ -0,0 +1,4 @@
+öüpäövrewbv
+
+erbgvöer45675)$%&§=)
+**’’¯¯

variable_grabbler.py

@@ -1,97 +1,113 @@
-#!/usr/bin/python2
+#!/usr/bin/env python3
 
-print """
-Variable Grabbler - version 4.0_pre6
-------------------------------------"""
-################################################################
-# Changes in this version:
-#===============================================================
-# *"\?" in a variable will now be replaced with a simple ?
-#  before processing, remember to double escape that because
-#  json doesn't like \? (=> \\?)
-# *commands are now not run over and over again if not needed
-# *files are now not rewritten for each variable individually
-# *some technical stuff
+import os, sys, json, traceback, subprocess
 
-################################################################
-# Exit codes:
-#===============================================================
-# 0 => normal exit
-# 1 => command line argument error
-# 2 => error while processing
-#
+# Definitions
+def stdout(string, end='\n', flush=False):
+    print(string, end=end, file=sys.stdout, flush=flush)
 
-# definitions
-import os, json, sys, subprocess
+def stderr(string, end='\n', flush=False):
+    print(string, end=end, file=sys.stderr, flush=flush)
 
-def print_err(text):
-  print >> sys.stderr, text
+exit_codes = {
+  "normal exit":            0,
+  "wrong usage":            1,
+  "error while processing": 2
+}
 
+stderr("Variable Grabbler - version 5.0_pre1\n--------------------------------------------------------------------------------")
+################################################################################
+# Chnages in this version:
+# - complete rewrite in Python 3
+# - output to stdout instead of rewriting the file
+#===============================================================================
+# Full documentation: tbd
+#===============================================================================
+
+# Command line input handling
+usage = "Usage: python3 "+sys.argv[0]+" <macro config file> <file to be processed>"
 if not len(sys.argv)==3:
-  print_err("Error: Exactly two arguments required: \"python variable_grabbler.py <file to be rewritten> <config file>\"")
-  sys.exit(1)
+    stderr("E: Wrong amount of arguments.")
+    stderr(usage)
+    sys.exit(exit_codes["wrong usage"])
+if sys.argv[1] == sys.argv[2]:
+    stderr("E: Both input files are the same.")
+    stderr(usage)
+    sys.exit(exit_codes["wrong usage"])
 
-print "Reading config file...",
-config_file = open(sys.argv[2],"r")
-config_values = json.loads(config_file.read().decode("utf-8"))
-config_file.close()
-print "Done."
-
-
-print "Reading file: "+sys.argv[1]+"..."
-file_in = open(sys.argv[1], "r")
-file_content = file_in.read()
-file_in.close()
-
-print "Replacing variables:"
-for variable in config_values:
-  print "> "+variable.upper()+" => ",
-
-  if config_values[variable][0] == "file":
-    print "Found array. Adding file instead: " + config_values[variable][1]
-    file_replacement = open(config_values[variable][1], "r")
-    file_content = file_content.decode('utf-8').replace("%" + variable.upper() + "%", file_replacement.read()).encode('utf-8')
-    file_replacement.close()
-  elif config_values[variable][0] == "exec":
-    print "Found array. Executing command instead: " + config_values[variable][1]
-    if variable.upper() in file_content:
-      process = subprocess.Popen(config_values[variable][1], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-      stdout, stderr = process.communicate()
-      if process.returncode == 0:
-        print "Process exited normally. Replacing variable with the output from STDOUT..."
-        print_err("Subprocess exited normally.\nThe following error messages were produced:")
-        print_err(stderr)
-        file_content = file_content.decode('utf-8').replace("%" + variable.upper() + "%", stdout).encode('utf-8')
-      else:
-        print "Process exited abnormally. Exiting. No changes will be made."
-        print_err("Subprocess exited abnormally. Exiting.\nThe following output was produced:")
-        print_err(stdout)
-        print_err("The following error messages were produced:")
-        print_err(stderr)
-        print "Exiting now."
-        sys.exit(2)
+# load macros into a dict
+stderr("I: Reading macro definitions... ")
+macros = {}
+try:
+    if os.path.isfile(sys.argv[1]):
+        macro_file = open(sys.argv[1], "r")
+        macros = json.loads(macro_file.read())
+        macro_file.close()
+    elif sys.argv[1] == '-':
+        macros = json.loads(sys.stdin.read())
     else:
-      print variable.upper(),
-      print "was not found in the file. Skipping command execution."
-  else:
-    replacement = str(config_values[variable])
-    # look for variable sections that contain question marks
-    argument_positions = []
-    i = 0
-    while i<len(replacement):
-      if replacement[i] == '?' and (i==0 or not replacement[i-1]=='\\'):
-        argument_positions = argument_positions + [ i ]
-      i = i+1
-    #print "DEBUG: " + str(argument_positions)
-    replacement = replacement.replace("\\?", "?")
-    print repr(replacement)
-    # actually replace variables
-    if argument_positions == []:
-      file_content = file_content.decode('utf-8').replace("%" + variable.upper() + "%", replacement).encode('utf-8')
+        stderr("E: Not a valid file: "+sys.argv[1])
+        sys.exit(exit_codes["error while processing"])
+except:
+    stderr("E: An exception occurred while trying to read macro definitions:")
+    traceback.print_exc()
+    sys.exit(exit_codes["error while processing"])
 
-print "Saving new file..."
-file_out = open(sys.argv[1], "w")
-file_out.write(file_content)
-file_out.close()
+# process "file" and "exec" macros
+stderr("I: Loading templates and processing commands...")
+for macro in macros:
+    if macros[macro][0] == "file":
+        if os.path.isfile(macros[macro][1]):
+            try:
+                stderr("I: Loading template: " + macro)
+                template = open(macros[macro][1], "r")
+                macros[macro] = template.read()
+                template.close()
+            except:
+                stderr("E: An exception occurred while trying to read the template:")
+                traceback.print_exc()
+                sys.exit(exit_codes["error while processing"])
+        else:
+            stderr("E: Macro \"" + macro + "\": template \"" + macros[macro][1] + "\" is not a valid file.")
+            sys.exit(exit_codes["error while processing"])
+    elif macros[macro][0] == "exec":
+        try:
+            stderr("I: Running command: " + macro)
+            process = subprocess.Popen(macros[macro][1], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            process_stdout, process_stderr = process.communicate()
+            macros[macro] = process_stdout
+            if len(process_stderr) > 0:
+                stderr("Output on stderr:\n"+process_stderr)
+            if not process.returncode == 0:
+                stderr("E: Command execution failed with exit code "+str(process.returncode))
+                sys.exit(exit_codes["error while processing"])
+        except:
+            stderr("E: An exception occurred while trying to process the command:")
+            traceback.print_exc()
+            sys.exit(exit_codes["error while processing"])
 
-print "Done."
+# open file to be processed and iterate over it
+stderr("I: Preparing to process input file...")
+try:
+    input_file = sys.stdin
+    if os.path.isfile(sys.argv[2]):
+        input_file = open(sys.argv[2], "r")
+    elif sys.argv[2] == '-':
+        # nothing to do here bc input_file is already set to sys.stdin
+        pass
+    else:
+        stderr("E: Not a valid file: "+sys.argv[2])
+        sys.exit(exit_codes["error while processing"])
+    # make the magic happen
+    stderr("I: Processing input file...")
+    for line in input_file:
+        for macro in macros:
+            line = line.replace("%" + macro + "%", str(macros[macro]))
+        stdout(line, end="")
+    # close input file if it's not sys.stdin, assume sys.stdin is being handled for us
+    if not input_file==sys.stdin:
+        input_file.close()
+except:
+    stderr("E: An exception occured while processing " + input_file.name + ":")
+    traceback.print_exc()
+    sys.exit(exit_codes["error while processing"])

variable_grabbler.py.old

@@ -0,0 +1,97 @@
+#!/usr/bin/python2
+
+print """
+Variable Grabbler - version 4.0_pre6
+------------------------------------"""
+################################################################
+# Changes in this version:
+#===============================================================
+# *"\?" in a variable will now be replaced with a simple ?
+#  before processing, remember to double escape that because
+#  json doesn't like \? (=> \\?)
+# *commands are now not run over and over again if not needed
+# *files are now not rewritten for each variable individually
+# *some technical stuff
+
+################################################################
+# Exit codes:
+#===============================================================
+# 0 => normal exit
+# 1 => command line argument error
+# 2 => error while processing
+#
+
+# definitions
+import os, json, sys, subprocess
+
+def print_err(text):
+  print >> sys.stderr, text
+
+if not len(sys.argv)==3:
+  print_err("Error: Exactly two arguments required: \"python variable_grabbler.py <file to be rewritten> <config file>\"")
+  sys.exit(1)
+
+print "Reading config file...",
+config_file = open(sys.argv[2],"r")
+config_values = json.loads(config_file.read().decode("utf-8"))
+config_file.close()
+print "Done."
+
+
+print "Reading file: "+sys.argv[1]+"..."
+file_in = open(sys.argv[1], "r")
+file_content = file_in.read()
+file_in.close()
+
+print "Replacing variables:"
+for variable in config_values:
+  print "> "+variable.upper()+" => ",
+
+  if config_values[variable][0] == "file":
+    print "Found array. Adding file instead: " + config_values[variable][1]
+    file_replacement = open(config_values[variable][1], "r")
+    file_content = file_content.decode('utf-8').replace("%" + variable.upper() + "%", file_replacement.read()).encode('utf-8')
+    file_replacement.close()
+  elif config_values[variable][0] == "exec":
+    print "Found array. Executing command instead: " + config_values[variable][1]
+    if variable.upper() in file_content:
+      process = subprocess.Popen(config_values[variable][1], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+      stdout, stderr = process.communicate()
+      if process.returncode == 0:
+        print "Process exited normally. Replacing variable with the output from STDOUT..."
+        print_err("Subprocess exited normally.\nThe following error messages were produced:")
+        print_err(stderr)
+        file_content = file_content.decode('utf-8').replace("%" + variable.upper() + "%", stdout).encode('utf-8')
+      else:
+        print "Process exited abnormally. Exiting. No changes will be made."
+        print_err("Subprocess exited abnormally. Exiting.\nThe following output was produced:")
+        print_err(stdout)
+        print_err("The following error messages were produced:")
+        print_err(stderr)
+        print "Exiting now."
+        sys.exit(2)
+    else:
+      print variable.upper(),
+      print "was not found in the file. Skipping command execution."
+  else:
+    replacement = str(config_values[variable])
+    # look for variable sections that contain question marks
+    argument_positions = []
+    i = 0
+    while i<len(replacement):
+      if replacement[i] == '?' and (i==0 or not replacement[i-1]=='\\'):
+        argument_positions = argument_positions + [ i ]
+      i = i+1
+    #print "DEBUG: " + str(argument_positions)
+    replacement = replacement.replace("\\?", "?")
+    print repr(replacement)
+    # actually replace variables
+    if argument_positions == []:
+      file_content = file_content.decode('utf-8').replace("%" + variable.upper() + "%", replacement).encode('utf-8')
+
+print "Saving new file..."
+file_out = open(sys.argv[1], "w")
+file_out.write(file_content)
+file_out.close()
+
+print "Done."