#!/bin/bash
#
# Copyright 2005 Marc Schiffbauer <marc@schiffbauer.net>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
#
# grtool is a small utility for grsecurity to manipulate policy files
# and convert learning logs etc.
#
# It might be useful for anybody creating and maintaining 
# grsecurity RBAC policies and RBAC learning logs
#

VERSION=0.2

#### FUNCTIONS
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

#*********************************************************************
function init_early () {
#*********************************************************************
  # set some esc sequences
  if test -t 1 -a "$TERM" != "raw" -a "$TERM" != "dumb" && stty size > /dev/null 2>&1 ; then
    esc=`echo -en "\033"`
    bold="${esc}[1m"
    red="${esc}[1;31m"
    green="${esc}[1;32m"
    yellow="${esc}[1;33m"
    norm=`echo -en "${esc}[m\017"`
  else
    esc=""
    bold=""
    red=""
    green=""
    yellow=""
    norm=""
  fi
  
  # stop on every error
  set -e
  
  # trap signals
  trap error_exit SIGHUP SIGINT SIGQUIT SIGABRT SIGTERM ERR
  trap clean_exit EXIT

  TMP_ID="$(mcookie)"
  TMP_ID="${TMP_ID:2:8}"
}

#*********************************************************************
function init_late () {
#*********************************************************************
  # init tmpdir
  mkdir -p -m 0700 "$TMPDIR"
}

#*********************************************************************
function set_defaults () {
#*********************************************************************
  # directory for temporary files
  TMPDIR="/tmp/$(basename $0).$TMP_ID"

  # buffer size for sort processes
  DEFAULT_SORT_BUFFER="5M"
}

#*********************************************************************
function get_tempfile () {
#*********************************************************************
  COOKIE="$(mcookie)"
  COOKIE="${COOKIE:0:10}"
  echo "$TMPDIR/$COOKIE"
}

#*********************************************************************
function check_env () {
#*********************************************************************
  # check if we find all commands we need
  CMDS="basename cat cp diff gawk kill mkdir mv nice rm sort test uniq getopt mcookie"
  MISSING=""
  for C in $CMDS; do
    echo_progress "Looking for '$C'"
    if which $C >/dev/null; then 
      confirm_ok "($(which $C))"
    else
      confirm_err "(not found)"
      MISSING="$MISSING $C"
    fi
  done 
  if [ "$MISSING" ]; then
    echo_error "Following command(s) cannot be found on your system: '$MISSING'. grtool needs it work properly."
  fi
}

#*********************************************************************
function echo_info () {
#*********************************************************************
  if [ "$AWAITING_STATUS" == "yes" ]; then
    confirm_ok
  fi
  test "$VERBOSE" && echo -e "$*"
  return 0
}

#*********************************************************************
function echo_warn () {
#*********************************************************************
  if [ "$AWAITING_STATUS" == "yes" ]; then
    confirm_ok
  fi
  echo -e "Warning: $*"
  return 0
}

#*********************************************************************
function echo_progress () {
#*********************************************************************
  if [ "$AWAITING_STATUS" == "yes" ]; then
    confirm_ok
  fi
  echo -en "$* ... "
  AWAITING_STATUS=yes
}

#*********************************************************************
function confirm_ok () {
#*********************************************************************
  if [ "$AWAITING_STATUS" == "yes" ]; then
    echo -e "${green}OK${norm} $*"
  fi
  AWAITING_STATUS=no
}

#*********************************************************************
function confirm_err () {
#*********************************************************************
  if [ "$AWAITING_STATUS" == "yes" ]; then
    echo -e "${red}ERROR${norm} $*"
  fi
  AWAITING_STATUS=no
  #error_exit
}

#*********************************************************************
function echo_error () {
#*********************************************************************
  if [ "$AWAITING_STATUS" == "yes" ]; then
    confirm_ok
  fi
  echo -e "${red}ERROR${norm}: $*" >/dev/stderr
  clean_exit 1
}

#*********************************************************************
function echo_version () {
#*********************************************************************
  OLD_VERBOSE=$VERBOSE
  VERBOSE=1
  echo_info "$(basename $0) - Version $VERSION"
  echo_info ""
  echo_info "Copright (C) 2005 Marc Schiffbauer"
  echo_info ""
  echo_info "This program is free software; you can redistribute it and/or modify"
  echo_info "it under the terms of the GNU General Public License as published by"
  echo_info "the Free Software Foundation; either version 2 of the License, or"
  echo_info "(at your option) any later version."
  echo_info ""
  echo_info "This program is distributed in the hope that it will be useful,"
  echo_info "but WITHOUT ANY WARRANTY; without even the implied warranty of"
  echo_info "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the"
  echo_info "GNU General Public License for more details."
  echo_info ""
  VERBOSE=$OLD_VERBOSE
}

#*********************************************************************
function echo_help () {
#*********************************************************************
  # print help and exit
  OLD_VERBOSE=$VERBOSE
  VERBOSE=1
  echo_info
  echo_info "Syntax: $(basename $0) [ options ] <command> <command parameters>"
  echo_info
  echo_info "Commands:"
  echo_info "  split-policy | s-p"
  echo_info "             Split a policy file generated by gradm"
  echo_info "             into several files and put them into a"
  echo_info "             directory structure"
  echo_info
  echo_info "  glob-objects | g-o"
  echo_info "             Automatically glob objects that match"
  echo_info "             a given regular expression."
  echo_info "             For example this is useful for changing"
  echo_info "             objects in /usr/lib or /lib with version"
  echo_info "             extension to a globbed object without"
  echo_info "             version to make software upgrades"
  echo_info "             possible without having to edit the policy"
  echo_info "             afterwards..."
  echo_info
#  echo_info "  convert-log | c-l"
#  echo_info "             Convert role or subject based learning logs"
#  echo_info "             to full learning logs so that they can be used"
#  echo_info "             for a full learning process"
#  echo_info
  echo_info "  shrink-log | s-l"
  echo_info "             Eliminate duplicate lines in a learning log to"
  echo_info "             make it smaller and speedup learning processes"
  echo_info "             and save diskspace."
  echo_info
  echo_info "Options:"
  echo_info ""
  echo_info "  -h|--help          Show help on commands and options"
  echo_info "  -c|--check-env     Check if all required unix tools can be found"
  echo_info "  -n|--nice          Nice 'heavy' subprocesses (useful on production"
  echo_info "                     systems to not produce too much load)"
  echo_info "  -N|--no-backups    Do not create backup files."
  echo_info "  -t|--temp-dir      Specify a directory to use for temporary files"
  echo_info "                     Default: /tmp/grtool.<random id>"
  echo_info "  -v|--verbose       Be verbose and show progress infos"
  echo_info "  -V|--version       Show version information"
  echo_info ""
  echo_info "  Try '$0 <command> --help' to get help on specific command parameters"
  echo_info ""
  VERBOSE=$OLD_VERBOSE
}

#*********************************************************************
function clean_exit () {
#*********************************************************************
  # fetch return code from last command causing this clean_ecit() call
  RC=$?

  if [ "$AWAITING_STATUS" == "yes" ]; then
    confirm_err
  fi
  test -d "$TMPDIR" && rm -rf "$TMPDIR"
  #echo_info "I will exit now"
  echo -en "${norm}"
  # prevent double execution
  trap EXIT

  if [ "$1" ]; then
    exit $1
  else
    exit $RC
  fi
}

#*********************************************************************
function error_exit () {
#*********************************************************************
  if [ "$AWAITING_STATUS" == "yes" ]; then
    confirm_err
  fi
  false
}

#*********************************************************************
function file_replace_string () {
#*********************************************************************
  # replace a string in a file
  TMP="$TMPDIR/$RANDOM"
  test -f "$TMP" && rm -f "$TMP"

  SEARCH_PATTERN="$1"
  REPLACE_PATTERN="$2"
  FILE="$3"

  if [ ! -f "$FILE" ]; then
    echo_error "$FILE does not exist"
    clean_exit 1
  fi

  # backup file
  cp -pRL "$FILE" "$TMPDIR/$(basename $FILE).bak"

  # do the replacements
  eval "sed 's/$SEARCH_PATTERN/$REPLACE_PATTERN/g' '$TMPDIR/$(basename $FILE).bak' > $FILE"

  # cleanup
  rm -f "$TMPDIR/${FILE}.bak"
}

#*********************************************************************
function process_args () {
#*********************************************************************
  # process command line arguments
  #
  # the main command will be stored in $COMMAND

  # print advice if no arguments are given
  if [ ! "$1" ]; then
    VERBOSE=1
    echo_info "Try $0 -h for help"
    clean_exit 1
  fi

  # seperate global options
  GLOBAL_OPTS=""
  while [ "$1" ]; do
    case "$1" in
#      split-policy|s-p|glob-objects|g-o|convert-log|c-l|shrink-log|s-l)
      split-policy|s-p|glob-objects|g-o|shrink-log|s-l)
        # this is a command, remaining opts are command specific
        COMMAND_OPT="$1"
        shift
        COMMAND_OPTS="$*"
        break
      ;;
      -*|--*)
        # save $1 as global opt
        GLOBAL_OPTS="$GLOBAL_OPTS $1"
      ;;
      *)
        # this mus be an argument to an opt
        GLOBAL_OPTS="$GLOBAL_OPTS $1"
    esac
    shift
  done

  GLOBAL_SHORT_OPTIONS="chnNt:vV"
  GLOBAL_LONG_OPTIONS="check-env,help,nice,no-backups,temp-dir:,verbose,version"

  # prepare opts
  GLOBAL_OPTS="$(getopt -u -o "$GLOBAL_SHORT_OPTIONS" -l "$GLOBAL_LONG_OPTIONS" -- $GLOBAL_OPTS)"

  # check global opts
  set -- $GLOBAL_OPTS
  while [ "$1" ]; do
    if [ "$1" == "--" ]; then
      shift
      continue
    fi
    case "$1" in
     -h|--help)
        echo_help
        clean_exit
      ;;
      -c|--check-env)
        echo_version
        check_env
        clean_exit
      ;;
      -n|--nice)
        NICE="nice"
        echo_info "All 'heavy' spawned processes will be niced"
      ;;
      -N|--no-backups)
        NO_BACKUPS="1"
        echo_info "No backups will be created."
      ;;
      -v|--verbose)
        VERBOSE=1
      ;;
      -t|--temp-dir)
        shift
        if [ ! -d "$1" ]; then
          echo_error "Temp dir '$1' is not a directory"
        fi
        TMPDIR="$1/$(basename $0.$TMP_ID)"
        echo_info "Tempfiles will be created in $TMPDIR"
      ;;
      -V|--version)
        echo_version
        clean_exit
      ;;
      -*|--*)
        echo_error "BUG: $1 not implemented. Please report to the author."
      ;;
    esac
    shift
  done

  # check $COMMAND
  test "$COMMAND_OPT" || echo_error "No command has been specified"
  case "$COMMAND_OPT" in
      split-policy|s-p)
        COMMAND="split_policy"
      ;;
      glob-objects|g-o)
        COMMAND="glob_objects"
      ;;
#      convert-log|c-l)
#        COMMAND="convert_log"
#      ;;
      shrink-log|s-l)
        COMMAND="shrink_log"
      ;;
      *)
        echo_error "Unknown command: $COMMAND_OPT"
      ;;
  esac     
}

#*********************************************************************
function split_policy () {
#*********************************************************************
  SHORT_OPTIONS="d:hp:"
  LONG_OPTIONS="policy:,destination:,help"

  # Set positional parameters to given arguments
  set -- $(getopt -u -o "$SHORT_OPTIONS" -l "$LONG_OPTIONS" -- "$@")

  while [ "$1" ]; do
    if [ "$1" == "--" ]; then
      shift
      continue
    fi
    case "$1" in
      -p|--policy)
        shift
        POLICY="$1"
        test -f "$1" || echo_error "Policy file '$1' not found"
      ;;
      -d|--destination)
        shift
        P_FILE="$1"
        P_DIR="$1.d"
        test -e "$P_FILE" && echo_error "Destination file '$P_FILE' already exists."
        test -e "$P_DIR"  && echo_error "Destination directory '$P_DIR' already exists."
      ;;
      -h|--help)
        VERBOSE=1
        echo_info "Mandatory parameters for split-policy:"
        echo_info ""
        echo_info "  -p|--policy <file>"
        echo_info "        Input policy file to split"
        echo_info "  -d|--destination <file>"
        echo_info "        Destination policy file"
        echo_info "        The directory structure with"
        echo_info "        split files will be build under"
        echo_info "        <file>.d and <file> is the master"
        echo_info "        policy file which only contains"
        echo_info "        include directives and can be used"
        echo_info "        as /etc/grsec/policy"
        echo_info "        Both, <file> and <file>.d  must not"
        echo_info "        exist before running this command."
        echo_info ""
        exit
      ;;
      -*)
        echo_error "BUG: $1 not implemented. Please report to the author."
      ;;
    esac
    shift
  done

  test "$POLICY" -a "$P_FILE" || echo_error "Please use -h to see mandatory parameters for split-policy"

  # FIXME: check if P_DIR exists

  # create policy directory
  mkdir -p $P_DIR

  # make P_DIR and absolute directory if its not yet
  # because include paths must not be relative
  if [ "${P_DIR:0:1}" != "/" ];then
    OLD_PWD="$(pwd)"
    cd "$P_DIR"
    P_DIR="$(pwd)"
    cd "$OLD_PWD"
  fi
  
  echo "# master policy file for split policies" > $P_FILE
  echo "# created $(date) by $(basename $0)" > $P_FILE
  echo "#" >> $P_FILE
  echo ""  >> $P_FILE
  
  IFS=""
  cat "$POLICY" | while read LINE; do
    # test for start of new role
  
    if [ "${LINE:0:5}" == "role " ]; then
      unset S_NAME_FILE
    
      R_TYPE="$(echo "$LINE"|cut -d' ' -f3)"
      R_NAME="$(echo "$LINE"|cut -d' ' -f2)"
 
      # determine role directory
      if [ "${R_TYPE:0:1}" == "u" ]; then
        R_DIR="$P_DIR/$R_NAME.user_role"
      elif [ "${R_TYPE:0:1}" == "g" ]; then
        R_DIR="$P_DIR/$R_NAME.group_role"
      elif [ "${R_TYPE:0:1}" == "s" ]; then
        R_DIR="$P_DIR$R_NAME.special_role"
      elif [ "$R_NAME" == "default" ]; then
        R_DIR="$P_DIR/default_role"
      fi
  
      echo_info "Creating policy directory $R_DIR"
      mkdir -p $R_DIR

      ROLE_FILE="_role"

      echo "include <$R_DIR/$ROLE_FILE>" >> $P_FILE
      echo "# role file with subject includes for '$R_NAME' - created by $(basename $0)" > $R_DIR/$ROLE_FILE
      echo "#" >> $R_DIR/$ROLE_FILE
      echo ""  >> $R_DIR/$ROLE_FILE
    fi
  
    if [ "${LINE:0:8}" == "subject " ]; then
      S_NAME="$(echo "$LINE"|cut -d' ' -f2)"

      # if S_NAME != "/" eliminate leading slash if there is one
      S_NAME="$(echo "$S_NAME" | sed 's|^/||')"
      
      # create path to subject file
      mkdir -p "$R_DIR/$(dirname "$S_NAME")"
      
      if [ "$S_NAME" ]; then
        S_NAME_FILE="${S_NAME}.subject"
      else
        S_NAME_FILE="default.subject"
      fi
      echo "include <$R_DIR/$S_NAME_FILE>" >> $R_DIR/$ROLE_FILE
      echo -n "" > "$R_DIR/$S_NAME_FILE"
      echo_info "Created new subject file subject $R_DIR/$S_NAME_FILE"
    fi
  
    if [ "$R_DIR" ]; then
      if [ "$S_NAME_FILE" ]; then
        echo "$LINE" >> $R_DIR/$S_NAME_FILE
      else
        echo "$LINE" >> $R_DIR/$ROLE_FILE
      fi
    fi
  done
}

#*********************************************************************
function glob_objects () {
#*********************************************************************

  SHORT_OPTIONS="dhp:r:R:s:"
  LONG_OPTIONS="dry-run,help,policy:,regex:,role:,subject:"

  # Set positional parameters to given arguments
  set -- $(getopt -u -o "$SHORT_OPTIONS" -l "$LONG_OPTIONS" -- "$@")

  while [ "$1" ]; do
    if [ "$1" == "--" ]; then
      shift
      continue
    fi
    case "$1" in
      -p|--policy)
        shift
        POLICY="$1"
        test -f "$1" || echo_error "Policy file '$POLICY' not found"
        if [ -z "$NO_BACKUPS" ]; then
          test -f "${POLICY}.bak" && echo_error "policy backup file $POLICY.bak already exists, remove it first"
        fi
      ;;
      -r|--regex)
        shift
        REGEX="$1"
      ;;
      -R|--role)
        shift
        ROLE="$1"
      ;;
      -s|--subject)
        shift
        SUBJECT="$1"
      ;;
      -d|--dry-run)
        DRYRUN=1
      ;;
      -h|--help)
        VERBOSE=1
        echo_info ""
        echo_info "Mandatory parameters for glob-objects:"
        echo_info ""
        echo_info "  -p|--policy <policy file>"
        echo_info "        Policy file to change."
        echo_info "        A backup of the original will be created as"
        echo_info "        <policy file>.bak if -N was not specified"
        echo_info ""
        echo_info "  -r|--regex <regular expression>"
        echo_info "        The regex must contain one or two pairs of braces"
        echo_info "        and the matching part within those will"
        echo_info "        be the part of the object that is preserved."
        echo_info "        The matching part within the first pair of braces"
        echo_info "        will be the part before the asterisk (*) and the"
        echo_info "        matching part of the second pair the one after it."
        echo_info ""
        echo_info "        Example:"
        echo_info "        -r '(/usr/lib/lib.*\.so\.).*'"
        echo_info "          will replace an object like"
        echo_info "            /usr/lib/libgnutls.so.11.1.16"
        echo_info "          by"
        echo_info "            /usr/lib/libgnutls.so.*"
        echo_info ""
        echo_info "        -r '(/lib/ld-2\.).*(\.so)'"
        echo_info "          will replace an object like"
        echo_info "            /lib/ld-2.2.5.so"
        echo_info "          by"
        echo_info "            /lib/ld-2.*.so"
        echo_info ""
        echo_info "Optional parameters for glob-objects:"
        echo_info ""
        echo_info "  -R|--role <role name>"
        echo_info "        Only process this role"
        echo_info ""
        echo_info "  -s|--subject <subject name>"  
        echo_info "        Only process this subject"
        echo_info ""
        echo_info "  -d|--dry-run"
        echo_info "        Testing mode. Do not change anything"
        echo_info ""
        exit
      ;;
      -*)
        echo_error "BUG: $1 not implemented. Please report to the author."
      ;;
    esac
    shift
  done

  test "$POLICY" -a "$REGEX" || echo_error "Please use -h to see mandatory parameters for glob-objects"

  # object format
  # <tab><object name><tab+><flags><\n>

  test "$ROLE" && echo_info "Selected role: $ROLE"
  test "$SUBJECT" && echo_info "Selected subject: $SUBJECT"
  if [ "$DRYRUN" ]; then
    echo_info "This is a dry-run, nothing will be changed."
    echo_info ""
    echo_info "But if you'd let me do I'd change the following:"
    IN="$POLICY"
    OUT="/dev/null"
  else
    IN="$POLICY"
    OUT="$(get_tempfile)"
    echo_info "Now looking for pattern $REGEX in policy file $IN ... "
  fi

  # gawk script to do the work
  $NICE gawk -v REGEX="$(echo "$REGEX" | sed 's/\\/\\\\/g')" \
    -v SUBJ="$SUBJECT" -v ROLE="$ROLE" -v DRYRUN="$DRYRUN" -v VERBOSE="$VERBOSE" '
        { 
          if (substr($1,1,4) == "role" && substr($1,1,5) != "role_")  {
            CROLE = $2;
            ALREADY_SHOWN = "no";
          }
          if (substr($1,1,7) == "subject")  {
            CSUBJ = $2;
            ALREADY_SHOWN = "no";
          }
          # only process if result is != input
          if ($1 != gensub(REGEX, "\\1*\\2", 1, $1)) {
            if (substr($1,1,1) == "/" && (CSUBJ == SUBJ || SUBJ == "") && (CROLE == ROLE || ROLE == "")) {
              if (match($1, REGEX) > 0) {
                if (ALREADY_SHOWN == "no") {
                  if (VERBOSE == "1") { print "\n"CROLE"::"CSUBJ":" >"/dev/stderr"; }
                  ALREADY_SHOWN = "yes";
                }
                if (VERBOSE == "1") { 
                  printf "\t" $1 >"/dev/stderr";
                  printf "\t" >"/dev/stderr";
                  for (i=1;i<(32-length($1))/8;i++) {
                    printf "\t" >"/dev/stderr";
                  }
                  printf "-->" >"/dev/stderr";
                  printf "\t" gensub(REGEX, "\\1*\\2", 1, $1) "\n" >"/dev/stderr";
                }
              }
            }
            if (substr($1,1,1) == "/" && (CSUBJ == SUBJ || SUBJ == "") && (CROLE == ROLE || ROLE == "") && DRYRUN == "") {
              if (match($1, REGEX) > 0) {
                a = gensub(REGEX, "\\1*\\2", 1, $1);
                printf "\t" gensub(REGEX, "\\1*\\2", 1, $1);  
                printf "\t";
                for (i=1;i<(32-length(a))/8;i++) {
                  printf "\t";
                }
                printf $2"\n";
              # else: regex did not match
              } else print $0;
            # else: $1 is not an object
            } else print $0; 
          # else: $1 == result after change
          } else print $0; 
        }
  ' "$IN" | uniq > "$OUT"

  if [ -z "$DRYRUN" ]; then
    # check if nothing has been changed
    if diff --brief "$IN" "$OUT" >/dev/null; then
      rm -f "$OUT"
    else
      if [ -z "$NO_BACKUPS" ]; then
        mv "$IN" "$IN.bak"
        echo_info ""
        echo_info "  $IN"
        echo_info "has been backed up as"
        echo_info "  $IN.bak"
        echo_info ""
      fi
      # overwrite original file with updated tmpfile
      echo_info "Writing new policy to '$IN' ... "
      mv -f "$OUT" "$IN"
    fi
  fi
}

#*********************************************************************
function convert_log () {
#*********************************************************************

  SHORT_OPTIONS="h"
  LONG_OPTIONS="help"

  # Set positional parameters to given arguments
  set -- $(getopt -u -o "$SHORT_OPTIONS" -l "$LONG_OPTIONS" -- "$@")

  while [ "$1" ]; do
    if [ "$1" == "--" ]; then
      shift
      continue
    fi
    case "$1" in
      -h|--help)
        VERBOSE=1
        echo_info ""
        echo_info "Mandatory parameters for convert-log:"
        echo_info ""
        echo_info "  <logfile>"
        echo_info "        The file will be converted so that it can be used for full"
        echo_info "        learning (gradm -F) policy generation."
        echo_info ""
        echo_info "        A backup copy of <logfile> will be saved to <logfile>.bak"
        echo_info "        (if -N was not specified)"
        echo_info ""
        exit
      ;;
      -*)
        echo_error "BUG: $1 not implemented. Please report to the author."
      ;;
      *)
        if [ -f "$1" ]; then
          LOG="$1"
        else
          echo_error "Logfile '$1' not found"
        fi
      ;;
    esac
    shift
  done

  test "$LOG" || echo_error "Please use -h for help on convert-log"
  if [ -z "$NO_BACKUPS" ]; then
    test -f "${LOG}.bak" && echo_error "logfile backup $LOG.bak already exists, remove it first"
  fi

  IN="$LOG"
  OUT="$(get_tempfile)"

  # look for cstream (to show progress)
  CSTREAM="$(which cstream || true)"
  PIDFILE="$(get_tempfile)"
  test -e "$PIDFILE" && echo_error "pid file for ctream already exists???"

  echo_info "Converting learning log '$IN' ... "
  echo_info ""

  if [ "$VERBOSE" -a -z "$CSTREAM" ]; then
    echo_info "You need to have the 'cstream' utility installed to make verbose mode work here"
    OLD_VERBOSE="$VERBOSE"
    unset VERBOSE
  fi
  
  if [ "$CSTREAM" -a "$VERBOSE" ]; then
    PCMD="cstream -l -p $PIDFILE"
    $NICE gawk '{
      FS="[\t]";
      $1="default";
      $6="/";
      #print $6 > "/dev/stderr";
      for (x=1;x<=NF;x++) {
        printf $x;
        if (x < NF) printf "\t";
      }
      printf "\n";
    }' "$IN" > "$OUT" &
    BGPID="$!"
    # wait for processes to be started but wait no longer than 3 seconds
    # which should be enough even for very busy machines but avoids a deadlock
    # for very small logfiles which will be processed very quick
    CNT=0
    until [ -f $PIDFILE.in -o "$CNT" -ge 3 ]; do
      sleep 1
      CNT=$((CNT+1))
    done
    while [ -f $PIDFILE ]; do
      kill -USR1 $(cat $PIDFILE)
      sleep 1
    done
    test -f "$PIDFILE" && rm "$PIDFILE"
  else
    $NICE gawk '{
      FS="[\t]";
      $1="default";
      $6="/";
      #print $6 > "/dev/stderr";
      for (x=1;x<=NF;x++) {
        printf $x;
        if (x < NF) printf "\t";
      }
      printf "\n";
    }' "$IN" > "$OUT"
  fi
  if [ "$VERBOSE" -a -z "$CSTREAM" ]; then
    VERBOSE="$OLD_VERBOSE"
  fi

  if [ -z "$NO_BACKUPS" ]; then
    echo_info "Backing up"
    echo_info "  $IN"
    echo_info "as"
    echo_info "  $IN.bak"
    echo_info ""
    mv "$IN" "$IN.bak"
  fi

  # write converted log
  echo_info "Writing converted log to '$IN' ..."
  mv "$OUT" "$IN"
  echo_info "finished"
}

#*********************************************************************
function __get_saved_percent () {
#*********************************************************************
  LOG_BIG="$1"
  LOG_SMALL="$2"

  SIZE_BIG="$(find $LOG_BIG -printf '%s')"
  SIZE_SMALL="$(find $LOG_SMALL -printf '%s')"

  SIZE_DIFF=$((SIZE_BIG-SIZE_SMALL))
  SIZE_DIFF_PERCENT=$((SIZE_DIFF*100/SIZE_BIG))
  echo -n "$SIZE_DIFF_PERCENT"
}

#*********************************************************************
function __get_saved_bytes () {
#*********************************************************************
  LOG_BIG="$1"
  LOG_SMALL="$2"

  SIZE_BIG="$(find $LOG_BIG -printf '%s')"
  SIZE_SMALL="$(find $LOG_SMALL -printf '%s')"

  SIZE_DIFF=$((SIZE_BIG-SIZE_SMALL))
  echo -n "$SIZE_DIFF"
}

#*********************************************************************
function shrink_log () {
#*********************************************************************

  SHORT_OPTIONS="hb:"
  LONG_OPTIONS="help,buffer-size:"

  # Set positional parameters to given arguments
  set -- $(getopt -u -o "$SHORT_OPTIONS" -l "$LONG_OPTIONS" -- "$@")

  while [ "$1" ]; do
    if [ "$1" == "--" ]; then
      shift
      continue
    fi
    case "$1" in
      -h|--help)
        VERBOSE=1
        echo_info ""
        echo_info "Mandatory parameters for shrink-log:"
        echo_info ""
        echo_info "  <logfile>"
        echo_info "        <logfile> can be any logfile produced by the grlearn daemon."
        echo_info "        The file will be shrinked by eliminating duplicate lines."
        echo_info "        A backup copy of <logfile> will be saved to <logfile>.bak"
        echo_info ""
        echo_info ""
        echo_info " Note:"
        echo_info "  To make --verbose work for this command you need to have"
        echo_info "  the 'cstream' utility installed"
        echo_info ""
        echo_info "Optional parameters for shrink-log:"
        echo_info ""
        echo_info "  -b|--buffer-size SIZE"
        echo_info "        use SIZE for main memory buffer of the sort process."
        echo_info ""
        echo_info "        See 'man sort' for details. Default is 5M."
        echo_info ""

        exit
      ;;
      -b|--buffer-size)
        shift
        SORT_BUFFER="$1"
      ;;
      -*)
        echo_error "BUG: $1 not implemented. Please report to the author."
      ;;
      *)
        if [ -f "$1" ]; then
          LOG="$1"
        else
          echo_error "Logfile '$1' not found"
        fi
      ;;
    esac
    shift
  done

  test "$LOG" || echo_error "Please use -h for help on shrink-log"
  if [ -z "$NO_BACKUPS" ]; then
    test -f "${LOG}.bak" && echo_error "logfile backup $LOG.bak already exists, remove it first"
  fi

  IN="$LOG"
  OUT="$(get_tempfile)"

  SIZE_ORIG="$(find $IN -printf '%s')"

  # set sort buffer if not set yet
  if [ -z "$SORT_BUFFER" ]; then
    SORT_BUFFER="$DEFAULT_SORT_BUFFER"
  else
    echo_info "Buffer size is $SORT_BUFFER"
  fi
  if $(echo "" | sort -S 5M >/dev/null 2>&1); then
    SORT="sort -u -T $TMPDIR -S $SORT_BUFFER"
  else
    echo_warn "Your 'sort' program does not seem to support custom buffersize settings"
    echo_warn "or you specified an invalid buffer size."
    echo_warn "This is not a problem, but the --buffer-size switch is useless"
    echo_warn "and shrinking a log may be slower."
    SORT="sort -u -T $TMPDIR"
  fi

  # look for cstream (to show progress)
  CSTREAM="$(which cstream || true)"
  PIDFILE="$(get_tempfile)"
  test -e "$PIDFILE" && echo_error "pid file for cstream already exists???"

  echo_info "Shrinking learning log '$IN' ... "
  echo_info ""
  if [ "$VERBOSE" -a -z "$CSTREAM" ]; then
    echo_info "You need to have the 'cstream' utility installed to make verbose mode work"
    OLD_VERBOSE="$VERBOSE"
    unset VERBOSE
  fi

  if [ "$VERBOSE" -a "$CSTREAM" ]; then
    PCMD_IN="cstream -l -p $PIDFILE.in"
    #PCMD_OUT="cstream -l -p $PIDFILE.out"
    #$NICE cat "$LOG.bak" | $PCMD_IN | $NICE $SORT | $PCMD_OUT > "$LOG" &
    $NICE cat "$IN" | $PCMD_IN | $NICE $SORT -o "$OUT" &
    BGPID="$!"
    # wait for processes to be started but wait no longer than 3 seconds
    # which should be enough even for very busy machines but avoids a deadlock
    # for very small logfiles which will be processed very quick
    CNT=0
    until [ -f $PIDFILE.in -o "$CNT" -ge 3 ]; do
      sleep 1
      CNT=$((CNT+1))
    done
    #while [ -f $PIDFILE.in -o -f $PIDFILE.out ]; do
    while [ -f $PIDFILE.in ]; do
      echo -n "Read "
      kill -USR1 $(cat $PIDFILE.in)
      sleep 1
    done
    echo_info "Reading finished."
    echo_info "Now eliminating duplicate loglines... this may take a while."
    wait
    test -f "$PIDFILE.in" && rm "$PIDFILE.in"
  else
    $NICE $SORT "$IN" >> "$OUT"
  fi
  if [ "$VERBOSE" -a -z "$CSTREAM" ]; then
    VERBOSE="$OLD_VERBOSE"
  fi
  echo_info ""
  echo_info "Done."
  echo_info "Saved $(__get_saved_bytes "$IN" "$OUT") Bytes ($(__get_saved_percent "$IN" "$OUT")%) of original logsize."

  if [ -z "$NO_BACKUPS" ]; then
    echo_info "Backing up"
    echo_info "  $IN"
    echo_info "as"
    echo_info "  $IN.bak"
    echo_info ""
    mv "$IN" "$IN.bak"
  fi

  # write converted log
  echo_info "Writing shrinked log to '$IN' ..."
  mv "$OUT" "$IN"
}

#### MAIN SCRIPT
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# do some early init stuff
init_early

# set some default values
set_defaults

# process command line arguments
process_args $*

# do some init stuff after processing command line arguments
init_late

# run the function specified in $COMMAND
$COMMAND $COMMAND_OPTS

exit 0
# end of script
#*********************************************************************
