#! /usr/bin/perl
# bogotune - a bogofilter tuning tool
# version 0.3, started 20030626
# Copyright (c) 2003 Gregory Louis; distributed wiithout warranty of
# any kind under the GNU General Public License (GPL).
use strict;

### 0.  Initialize and process arguments
use POSIX;

# choose apclass (0) or bogofilter (1) for scanning; apclass is twice
# as fast, if you have it, but it's not yet freely available.
my $usebogofilter = ($ENV{"IRCNICK"} ne "bigglebai");

# $bogop may be an absolute path to bogofilter and bogoutil, eg
# $bogop = "/newversion/"; # for testing
my $bogop = "";
my $bogof = $bogop . "bogofilter"; my $bogou = $bogop . "bogoutil";

# besides being human-readable, this help text is formatted for automatic
# man-page generation by the txt2man utility (http://mvertes.free.fr);
# don't change the format unless you know what you're doing!
if(scalar @ARGV == 0 || $ARGV[0] eq "-h") {
    print("NAME\n",
	"  bogotune - find optimum parameter settings for bogofilter\n",
	"SYNOPSIS\n",
	"  bogotune [bogodir] [-v] -s spamfile [spamfile...]",
	" -n okfile [okfile...]\n",
	"  bogotune -h\n",
	"DESCRIPTION\n",
	"  bogotune determines optimum parameter settings for bogofilter\n",
	"  using the production training database and one or more files of\n",
	"  spam messages, plus one or more files of nonspam. The training\n",
	"  database must have been built from at least 2,000 spam and 2,000\n",
	"  nonspam messages, and the ratio of spam to nonspam must be in\n",
	"  the range 0.2 to 5. There must be at least 500 spam messages\n",
	"  and 500 nonspam in the message files. Message files must all\n",
	"  be in MH, mbox or message-count format; don't mix files of\n",
	"  different types.\n",
	"COMMAND LINE PARAMETERS\n",
	"  bogodir   The directory where the training database is stored.\n",
	"	     bogodir defaults to \$BOGOFILTER_DIR, if defined, or\n",
	"	     otherwise to ~/.bogofilter\n",
	"  -v	List scan output in detail instead of using a progress\n",
	"	     meter.\n",
	"  -s	Following parameters are absolute paths to message\n",
	"	     files containing spam.\n",
	"  -n	Following parameters are absolute paths to nonspam\n",
	"	     files.\n",
	"  -h	Display help.\n",
	"  -C   Tell bogofilter not to use any configuration file.\n",
	"  -c   Following parameter is the absolute path to a\n",
	"            configuration file that bogofilter should use.\n",
	"BUGS\n",
	"  bogotune is not particularly robust when presented with\n",
	"  garbage input.\n",
	"SEE ALSO\n",
	"  bogofilter(1), bogolexer(1), bogoutil(1), apclass(1),\n",
	"  bogol(1), README.bogotune.\n\n");
    exit(0);
}

my $state = my $verbose = 0; my $cf = "";
my ($bogodir, @spfiles, @nsfiles);
foreach my $arg (@ARGV) {
       if($arg eq "-s") { $state = 2; }
    elsif($arg eq "-n") { $state = 3; }
    elsif($arg eq "-C") { $cf = "-C"; }
    elsif($arg eq "-c") { $state = 4; }
    elsif($arg eq "-v") { $verbose = 1; }
    elsif($state == 0 ) { $bogodir = $arg; $state = 1; }
    elsif($state == 2 ) { push @spfiles, $arg; }
    elsif($state == 3 ) { push @nsfiles, $arg; }
    elsif($state == 4 ) { $cf = "-c " . $arg; $state = 0; }
}

print "Using apclass.\n" unless $usebogofilter or ! $verbose;
my $workfn = "bt" . $$;

### 1.  Validate bogofilter database
# Check that $bogodir contains a valid training database,
# consisting of not fewer than 2000 each of spam and nonspam,
# and a ratio of spam to nonspam that lies in [0.2,5].

sub yuk {
    my $retcode = shift @_;
    my $str=join " ", @_; print $str, "\n\n";
    unlink $workfn; exit($retcode);
}

if(!$bogodir) {
    $bogodir = $ENV{"BOGOFILTER_DIR"};
    if($bogodir eq "") {
	$bogodir = $ENV{"HOME"} . "/.bogofilter";
    }
}
my $dbname = $bogodir . "/wordlist.db";
my $twolist = 0;
if(! -r $dbname) {
    $dbname = $bogodir . "/goodlist.db";
    $twolist = 1;
}
print("Verifying training db ", $dbname, "...\n");
my $counts;
open(COUNTS,"$bogou -w" . $bogodir . " .MSG_COUNT |") or
    yuk(255, "Couldn't run bogoutil");
while(<COUNTS>) { $counts = $_; }
chop $counts; close COUNTS;
my $err = 0; my ($scount, $ncount);
if(!$counts) { $err = 1; }
else {
    ($scount, $ncount) = (split /\s+/, $counts)[1,2];
    if($scount < 2000 || $ncount < 2000) { $err = 2; }
    else {
	my $scalefactor = $scount / $ncount;
	if($scalefactor < 0.2 || $scalefactor > 5) { $err = 3; }
    }
}
if($err > 0) {
    my @errstr = ("No wordlist found",
	"At least 2000 spam and 2000 nonspam required in training db",
	"Wordlist is out of balance");
    yuk($err, $errstr[$err-1]);
}
system("db_verify", $dbname) == 0 or yuk(6, "Verification failed!");
if($twolist) {
    print("Verifying training db ", $bogodir, "/spamlist.db...\n");
    system("db_verify", $bogodir . "/spamlist.db") == 0 or
	yuk(6, "Verification failed!");
}

### 2.  Validate test inputs; create msg-count files if need be
# Check that at least 500 each of spam and nonspam are available for
# testing, in MH, mbox or msg-count format.

print("Verifying test files...\n");
my $msgformat = "mbox";
if(-d $spfiles[0]) {
    $msgformat = "MH"; $scount = 0;
    foreach my $dir (@spfiles) {
	$scount += `find $dir -type f -name '[0-9]*' 2>/dev/null | wc -l`;
    }
    $ncount = 0;
    foreach my $dir (@nsfiles) {
	$ncount += `find $dir -type f -name '[0-9]*' 2>/dev/null | wc -l`;
    }
} else {
    my $cmd = join(" ", "cat", @spfiles, "| grep -c '^From ' |");
    open(COUNTS, $cmd) or yuk(5, "$cmd pipe failed");
    $scount = <COUNTS>; chop $scount; close COUNTS;
    $cmd = join(" ", "cat", @nsfiles, "| grep -c '^From ' |");
    open(COUNTS, $cmd) or yuk(5, "$cmd pipe failed");
    $ncount = <COUNTS>; chop $ncount; close COUNTS;
    if($scount < 500 || $ncount < 500) {
	$msgformat = "msgcount";
	$cmd = join(" ", "cat", @spfiles, "| grep -c '^.\.MSG_COUNT' |");
	open(COUNTS, $cmd) or yuk(5, "$cmd pipe failed");
	$scount = <COUNTS>; chop $scount; close COUNTS;
	$cmd = join(" ", "cat", @nsfiles, "| grep -c '^.\.MSG_COUNT' |");
	open(COUNTS, $cmd) or yuk(5, "$cmd pipe failed");
	$ncount = <COUNTS>; chop $ncount; close COUNTS;
    }
}
if($scount < 500 || $ncount < 500) { yuk(4,
    "At least 500 spam and 500 nonspam required for testing\n",
    $scount, " and ", $ncount, " were found."); }
print("Verification completed successfully.\n");
if($msgformat ne "msgcount") {
    print("Creating message-count files...\n");
    my $spwork = $workfn . ".sp"; my $nswork = $workfn . ".ns";
    if($msgformat eq "mbox") {
	my $cmd = join(" ", "cat", @spfiles, "| formail -es bogol",
	    $bogodir, $cf, ">", $spwork);
	system($cmd) == 0 or yuk(7, "Problem processing spam files");
	$cmd = join(" ", "cat", @nsfiles, "| formail -es bogol", $bogodir,
	    $cf, ">", $nswork);
	system($cmd) == 0 or yuk(7, "Problem processing nonspam files");
    } else {
	unlink($spwork);
	foreach my $dir (@spfiles) {
	    open FL, "find $dir -type f -name '[0-9]*'|" or yuk(7, "Problem processing spam files");
	    foreach my $msg (<FL>) {
		chomp($msg);
		my $cmd = join(" ", "cat $msg | bogol", $bogodir,
			$cf, ">>$spwork");
		system($cmd) == 0 or
			yuk(7, "Problem writing spam msg-count file");
	    }
	}
	unlink($nswork);
	foreach my $dir (@nsfiles) {
	    open FL, "find $dir -type f -name '[0-9]*'|" or yuk(7, "Problem processing nonspam files");
	    foreach my $msg (<FL>) {
		chomp($msg);
		my $cmd = join(" ", "cat $msg | bogol", $bogodir,
			$cf, ">>$nswork");
		system($cmd) == 0 or
			yuk(7, "Problem writing nonspam msg-count file");
	    }
	}
    }
    @spfiles = $spwork; @nsfiles = $nswork;
    print("Message-count files ", $workfn, ".{sp,ns} created\n");
}

### 3.  Calculate cache size
# Calculate the cache size that should be used.

my $cmd = "ls -l " . $dbname . " |";
open(COUNTS, $cmd) or yuk(5, "$cmd pipe failed");
$counts = <COUNTS>; chop $counts; close COUNTS;
my $dbsize = (split(/\s+/, $counts))[4];
if($twolist) {
    my $cmd = "ls -l " . $bogodir . "/spamlist.db |";
    open(COUNTS, $cmd) or yuk(5, "$cmd pipe failed");
    $counts = <COUNTS>; chop $counts; close COUNTS;
    my $dbs2 = (split(/\s+/, $counts))[4];
    if($dbs2 > $dbsize) { $dbsize = $dbs2; }
}
my $cachesize = POSIX::ceil($dbsize / (1024*1024*3));
print("Recommended cache size is ", $cachesize, " Mbytes.\n");

### 4.  Calculate fp target
# The fp target will be derived thus: score nonspams with s and md as
# shipped, and determine the count that will result from a spam cutoff
# of 0.95; if that is < 0.25%, try 0.9375 etc.

print("Calculating false-positive target...\n");

if($usebogofilter) {
    $cmd = join(" ", "cat", @nsfiles, "|", $bogof, "-m 0.02", $cf,
	"-d", $bogodir, "-Mttv >", $workfn);
} else {
    $cmd = join(" ", "cat", @nsfiles, "| apclass -fq -m 0.02 >", $workfn);
}
$err = system($cmd) >> 8;
if($err > 2) { print("Warning re processing nonspam messages: ", $err, "\n"); }
open(SCORES, $workfn) or yuk(8, "Couldn't open nonspam scores");
my $target = 0; my $thresh = 0.95; my @nsscores;
while(<SCORES>) {
    chop; my ($V1, $V2) = split(/\s+/);
    if($V2 == "") { $V2 = $V1; } elsif($V1 == 1) { $V2 = 1 - $V2; }
    push @nsscores, $V2;
    if($V2 > $thresh) { ++$target; }
}
close SCORES; --$target;
while($target < POSIX::ceil(scalar @nsscores * 0.0025)) {
    $thresh -= 0.0125;
    if ($thresh < 0.5) {
	print("Very few high-scoring nonspams in this data set.\n",
	    "Use these settings (only min_dev may have changed):\n");
	open(SCORES, "$bogof $cf -d $bogodir -Q |"); my $n = 0;
	while(<SCORES>) {
	    ++$n;
	    if($n == 6) { print "min_dev     = 0.020000 (2.00e-02)\n"; }
	    elsif($n > 3) { print($_); }
	    last if($n >= 8);
	}
	close SCORES;
	yuk(0, "Tuning aborted.")
    }
    $target = 0;
    foreach my $n (@nsscores) {
	if($n > $thresh) { ++$target; }
    }
}
print("False-positive target is ", $target, " (cutoff ", $thresh, ")\n");

### 5.  Calculate x
# Calculate x with bogoutil's -r option (a new addition).
# Bound the calculated value within [0.4, 0.6] and set the range to be
# investigated to [x-0.1, x+0.1].

print("Calculating initial x value...\n");
open(SCORES, "$bogou -r " . $bogodir . " |") or
    yuk(9, "Couldn't run bogoutil");
my $robx = <SCORES>; chop $robx; close SCORES;
$robx = 0.6 if $robx > 0.6; $robx = 0.4 if $robx < 0.4;
my $x0 = $robx - 0.1; my $x1 = $robx + 0.1;
printf("Initial x value is %0.6f\n", $robx);

### 6.  Coarsely scan s, md and x
# The coarse s scan will range from 1 to 0.01 in half decades, and the
# coarse md scan will range from 0.05 to 0.45 in steps of 0.05.  The
# coarse x scan will use steps of 0.05. The trough must be surrounded on
# six sides by values below the 33% quantile (unless bounded on one or
# more sides).

sub progress {
    my ($cur, $top) = @_;
    my $i; my $ndots = POSIX::ceil(70 * $cur / $top);
    if($ndots < 1) { $ndots = 1; }
    print("\r["); for $i (0 .. $ndots-1) { print("."); }
    for($i = $ndots; $i < 70; ++$i) { print(" "); }
    print("]");
}

sub col5 { $$a[5] <=> $$b[5]; }

my(@mdval,@rxval,@parms);

sub gfn {
    my ($si, $mi, $xi) = @_; my $pi;
    $pi = $si * ($#mdval+1) * ($#rxval+1) + $mi * ($#rxval+1) + $xi;
    return($parms[$pi][5]);
}

sub my_log10 { my $n = shift; return log($n) / log(10); }

my @rsval = (10 ** 0, 10 ** -0.5, 10 ** -1, 10 ** -1.5, 10 ** -2);
@mdval = (0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45);
my $n = ($x1 - $x0);
@rxval = ($x0 + $n/2, $x0 + $n/4, $x0 + $n*0.75, $x0, $x1);
my ($robs, $md);
foreach my $scantype ("coarse", "fine") {
    my ($rsi, $mdi, $rxi, $cutoff, $fn);
    print("Performing ", $scantype, " scan:\n");
    my $ord = 0; my $np = scalar @rsval * scalar @mdval * scalar @rxval;
    @parms = ();
    foreach $rsi (0 .. $#rsval) {
	foreach $mdi (0 .. $#mdval) {
	    foreach $rxi (0 .. $#rxval) {
		my $mpar = $mdval[$mdi] . "," . $rsval[$rsi] . "," .
		    $rxval[$rxi];
		### first get cutoff
		if($usebogofilter) {
		    $cmd = join(" ", "cat", @nsfiles, "|", $bogof, $cf, "-d",
			$bogodir, "-Mttv -m", $mpar, "-o 0.01 >", $workfn);
		} else {
		    $cmd = join(" ", "cat", @nsfiles, "| apclass -fq -m",
			$mpar, "-o 0.01 >", $workfn);
		}
		$err = system($cmd) >> 8;
		if($err > 2) {
		    print("Warning re processing nonspam messages: ",
			$err, "\n");
		}
		open(SCORES, $workfn) or
		    yuk(8, "Couldn't open nonspam scores");
		my @scores = ();
		while(<SCORES>) {
		    chop; my ($V1, $V2) = split(/\s+/);
		    if($V2 eq "") { push @scores, 1-$V1; }
		    elsif($V1 == 1) { push @scores, $V2; }
		}
		close SCORES;
		@nsscores = sort { $a <=> $b } @scores;
		my $fp = $target;
		if(scalar @nsscores >= $fp) {
		    while($nsscores[$fp-1] < 0.000001) { ++$fp; }
		    $cutoff = 1 - $nsscores[$fp-1];
		} else { yuk(10,
		    "Too few false positives to determine a valid cutoff");
		}
		### now get fn count
		if($usebogofilter) {
		    my $cmd = join(" ", "cat", @spfiles, "|", $bogof, $cf,
			"-d", $bogodir, "-Mttv -m", $mpar, "-o", $cutoff,
			">", $workfn);
		    $err = system($cmd) >> 8;
		    if($err > 2) {
			print("Warning re processing spam messages: ",
			    $err, "\n");
		    }
		    open(SCORES, $workfn) or
			yuk(8, "Couldn't open spam scores");
		    $fn = 0;
		    while(<SCORES>) {
			chop; my ($V1, $V2) = split(/\s+/);
			if($V2 eq "") { ++$fn if $V1 < $cutoff; }
			else { ++$fn if $V1 != 1; }
		    }
		    close SCORES;
		} else {
		    my $cmd = join(" ", "cat", @spfiles, "| apclass -fq -m",
			$mpar, "-o", $cutoff, "| grep -c -v '^1' |");
		    open(COUNTS, $cmd) or yuk(5, "$cmd pipe failed");
		    $fn = <COUNTS>; chop $fn; close COUNTS;
		}
		my @parm = ($rsi, $mdi, $rxi, $cutoff, $fp, $fn);
		push @parms, [ @parm ];
		if($verbose) {
		    printf("%0.4f %0.3f %0.3f %0.6f %0.0f %0.0f\n",
                        $rsval[$rsi], $mdval[$mdi], $rxval[$rxi], $cutoff,
                        $fp, $fn);
		} else {
		    progress($ord, $np);
		}
		++$ord;
	    }
	}
    }
    if(! $verbose) { print "\r"; }
    # Scan complete, now find minima
    my @unsorted = ();
    foreach my $row (@parms) {
	push @unsorted, [ @$row ] if $$row[4] == $target;
    }
    if(scalar @unsorted == 0) {
	@unsorted = @parms;
	print("Warning: fp target was not met, using original results\n");
    }
    my @sorted = sort col5 @unsorted;
    if($verbose) {
	print("Top ten parameter sets from this scan:\n",
	    "    rs    md    rx    co   %fp    %fn\n");
	foreach my $i (0 .. 9) {
	    ($rsi, $mdi, $rxi, $cutoff, my $fp, $fn) = @{$sorted[$i]};
	    printf("%6.4f %5.3f %5.3f %5.3f %5.3f %6.3f\n",
		$rsval[$rsi], $mdval[$mdi], $rxval[$rxi], $cutoff,
		100 * $fp / $ncount, 100 * $fn / $scount);
	}
    }
    my $med = $sorted[int(scalar @sorted / 4)][5];
    if($verbose) { print("25% fn count was ", $med, "\n"); }
    my $n = my $o = 0;
    foreach my $i (0 .. $#sorted) {
	$rsi = $sorted[$i][0]; $mdi = $sorted[$i][1]; $rxi = $sorted[$i][2];
	if(    ($rsi == 0       || gfn($rsi-1, $mdi, $rxi) < $med)
	    && ($rsi == $#rsval || gfn($rsi+1, $mdi, $rxi) < $med)
	    && ($mdi == 0       || gfn($rsi, $mdi-1, $rxi) < $med)
	    && ($mdi == $#mdval || gfn($rsi, $mdi+1, $rxi) < $med)
	    && ($rxi == 0       || gfn($rsi, $mdi, $rxi-1) < $med)
	    && ($rxi == $#rxval || gfn($rsi, $mdi, $rxi+1) < $med)) {
	    $n = 1; last;
	}
	$o = $i+1;
    }
    if($o > 0) {
	 print($o, " outlier", $o > 1 ? "s" : "", " encountered.",
	"                                                   \n");
    }
    if($n == 0) {
	$rsi = $sorted[0][0]; $mdi = $sorted[0][1]; $rxi = $sorted[0][2];
	print("No smooth minimum encountered, using lowest fn count",
	    " (an outlier).         \n");
    }
    $robs = $rsval[$rsi]; $md = $mdval[$mdi]; $robx = $rxval[$rxi];
    printf("Minimum found at s %0.4f, md %0.3f, x %0.3f %s\n",
	$robs, $md, $robx, "                           ");

    ### 7.  Finely scan the peak region
    # The fine s scan will range over the estimated s +/- half a decade in
    # steps of a quarter decade, and the fine md scan will range over the
    # estimated md +/- 0.075 in steps of 0.015.  The fine x scan will range
    # over the estimated x +/- 0.04 in steps of 0.02.  Scans of s and md
    # are bounded by the limits of the coarse scan.  Again, the troubh must
    # be surrounded on six sides by values below the 33% quantile.  If no
    # such trough exists, a warning is given.

    if($scantype == "coarse") {
	@rsval = @mdval = ();
	my $s0 = my_log10($robs) - 0.5; $s0 = -2 if $s0 < -2;
	my $s1 = my_log10($robs) + 0.5; $s1 = 0 if $s1 > 0;
	for(my $i=$s1; $i>=$s0; $i-=0.25) { push @rsval, 10 ** $i; }
	$s0 = $md - 0.075; $s0 = 0.02 if $s0 < 0.02;
	$s1 = $md + 0.075; $s1 = 0.465 if $s1 > 0.465;
	for(my $i=$s0; $i<=$s1; $i+=0.015) { push @mdval, $i; }
	@rxval = ($robx, $robx-0.02, $robx+0.02, $robx-0.04, $robx+0.04);
    }
}

### 8.  Suggest possible spam and nonspam cutoff values
# With the final x, md and s values, score the spams and nonspams and
# sort the nonspam scores decreasing and the spam scores increasing;
# then, traverse the nonspam list until the 0.2% point; report cutoffs
# that give 0.05%, 0.1% and 0.2% fp.

print("Performing final scoring:\nNonspam...\n");
my $mpar = $md . "," . $robs . "," . $robx;
if($usebogofilter) {
    $cmd = join(" ", "cat", @nsfiles, "|", $bogof, $cf, "-d",
	$bogodir, "-Mttv -m", $mpar, ">", $workfn);
} else {
    $cmd = join(" ", "cat", @nsfiles, "| apclass -fq -m",
	$mpar, ">", $workfn);
}
$err = system($cmd) >> 8;
if($err > 2) {
    print("Warning re processing nonspam messages: ", $err, "\n");
}
open(SCORES, $workfn) or yuk(8, "Couldn't open nonspam scores");
my @scores = ();
while(<SCORES>) {
    chop; my ($V1, $V2) = split(/\s+/);
    if($V2 == "") { $V2 = $V1; }
    elsif($V1 == 1) { $V2 = 1 - $V2; }
    push @scores, $V2;
}
close SCORES;
@nsscores = sort { $b <=> $a } @scores;
print("Spam...\n");
if($usebogofilter) {
    $cmd = join(" ", "cat", @spfiles, "|", $bogof, $cf, "-d",
        $bogodir, "-Mttv -m", $mpar, ">", $workfn);
} else {
    $cmd = join(" ", "cat", @spfiles, "| apclass -fq -m",  
        $mpar, ">", $workfn);
}
$err = system($cmd) >> 8;
if($err > 2) {
    print("Warning re processing spam messages: ", $err, "\n");
}
open(SCORES, $workfn) or yuk(8, "Couldn't open spam scores");
@scores = ();
while(<SCORES>) {
    chop; my ($V1, $V2) = split(/\s+/);
    if($V2 == "") { $V2 = $V1; }
    elsif($V1 == 1) { $V2 = 1 - $V2; }
    push @scores, $V2;
}
close SCORES;
my @spscores = sort { $a <=> $b } @scores;
print("Recommendations:\n\n---cut---\n",
    "db_cachesize=", $cachesize, "\n");
printf("robx=%0.6f\nmin_dev=%0.3f\nrobs=%0.4f\n", $robx, $md, $robs);
my $printed = 0;
foreach my $minn (10000, 2000, 1000, 500) {
    if($ncount > $minn) {
	my $cutoff = sprintf("%0.3f", $nsscores[POSIX::ceil($ncount /
	    $minn)-1]);
	if($cutoff < 0.999) {
            my $n = $#spscores;
	    for my $i (0 .. $#spscores) {
		if($spscores[$i] > $cutoff) { $n=$i; last; }
	    }
	    my $fneg = sprintf("%0.2f", 100 * $n / $scount);
	    print("#") if ($printed == 1);
	    print("spam_cutoff=", $cutoff, "\t# for ", 100 / $minn,
		"% false positives; expect ", $fneg, "% false neg.\n");
	    $printed=1;
	}
    }
}
if($printed == 0) {
    for my $i (0 .. $#nsscores)
	{ if($nsscores[$i] <= 0.999) { $target=$i; last; }}
    my $cutoff = sprintf("%0.3f", $nsscores[$target]); my $n = $#spscores;
    for my $i (0 .. $#spscores)
	{ if($spscores[$i] > $cutoff) { $n=$i; last; }}
    my $fneg = sprintf("%0.2f", 100 * $n / $scount);
    my $fp = sprintf("%0.2f", 100 * $target / $ncount);
    print("spam_cutoff=", $cutoff, "\t# for ", $fp, "% false positives; expect ",
	$fneg, "% false neg.\n");
}
my $cutoff = sprintf("%0.3f", $spscores[POSIX::ceil($scount * 0.002)-1]);
$cutoff = 0.1 if $cutoff < 0.1; $cutoff = 0.4 if $cutoff > 0.4;
print("ham_cutoff=", $cutoff, "\n---cut---\n");
yuk(0, "\nTuning completed.");
