#!/usr/bin/env perl
# -*- coding: ascii -*-
###########################################################################
# clive, the non-interactive video extraction utility
#
# Copyright (c) 2007-2009 Toni Gundogdu <legatvs@gmail.com>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
# 
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
###########################################################################

use warnings;
use strict;

binmode(STDOUT, ":utf8");

use URI::Escape qw(uri_escape uri_unescape);
use XML::Simple qw(XMLin);
use WWW::Curl::Easy 4.05;
use HTML::TokeParser;
use Config::Tiny;
use BerkeleyDB;

# Core modules:
use Getopt::Long qw(:config bundling);
use File::Basename qw(basename);
use Digest::SHA qw(sha1_hex);
use Pod::Usage qw(pod2usage);
use File::Path qw(mkpath);
use POSIX qw(strftime);
use Cwd qw(getcwd);
use File::Spec;
use Encode;

use constant VERSION             => "2.1.6";
use constant MBDIV               => 0x100000;
use constant BP_REFRESH_INTERVAL => 0.2;
use constant SHOWFMT_DEFAULT     => qq/%D: "%t" | %mMB/;

# Non-essential modules: set flags indicating availability
my %opted_mods = (Clipboard => 1, Expect => 1, IOPager => 1);
eval "use Clipboard;"; $opted_mods{Clipboard}   = 0 if $@;
eval "use IO::Pager;"; $opted_mods{IOPager}     = 0 if $@;
sub exp_continue() {}; # Satisfies: "Bareword "exp_continue" not allowed while"
eval "use Expect;"; $opted_mods{Expect}         = 0 if $@;

my $CONFIGDIR   = $ENV{CLIVE_HOME}
    || File::Spec->catfile($ENV{HOME}, ".config/clive");

my $CONFIGFILE  = File::Spec->catfile($CONFIGDIR, "config");
my $CACHEFILE   = File::Spec->catfile($CONFIGDIR, "cache");
my $RECALLFILE  = File::Spec->catfile($CONFIGDIR, "recall");

my %opts;           # runtime options
my @queue;          # input URLs
my $curl;           # curl handle, reused throughout lifespan
my $cache_db;       # handle to cache BDB
my %cache;          # handle to cache BDB (tied hash)
my $hash;           # sha1 hash of the current url used together with %cache
my %entry;          # multi-purpose hash for caching
my $ytube_logged=0; # youtube: whether logged-in
my $time_started;   # time file transfer started
my @exec_files;     # holds fnames for --exec
my @emit_queue;     # videos to be emitted
my $logfile;        # path to logfile (--output-file, --append-file)
my %dp;             # dot progress data
my %bp;             # bar progress data
my $workdir=getcwd; # startup workdir

my %re_hosts = (    # Precompiled regex used to identify the host
    IsYoutube   => qr|\Qyoutube.com\E|i,
    IsGoogle    => qr|\Qvideo.google.\E|i,
    IsSevenload => qr|\Qsevenload.com\E|i,
    IsBreak     => qr|\Qbreak.com\E|i,
    IsLastfm    => qr|\Qlast.fm\E|i,
    IsLiveleak  => qr|\Qliveleak.com\E|i,
    IsEvisor    => qr|\Qevisor.tv\E|i,
);

# Parse config
my $c = Config::Tiny->read($CONFIGFILE);
%opts = (
    progress => $c->{_}->{progress},
    agent    => $c->{http}->{agent},
    proxy    => $c->{http}->{proxy},
    maxspeed => $c->{http}->{maxspeed},
    minspeed => $c->{http}->{minspeed},
    savedir  => $c->{output}->{savedir},
    cclass   => $c->{output}->{cclass},
    fnfmt    => $c->{output}->{file},
    showfmt  => $c->{output}->{show},
    ytuser   => $c->{youtube}->{user},
    ytpass   => $c->{youtube}->{pass},
    clivepass=> $c->{commands}->{clivepass},
);

$opts{clivepass}    = $ENV{CLIVEPASS_PATH} unless $opts{clivepass};
$opts{progress}     = 'bar' unless $opts{progress};
$opts{format}       = 'flv';
$opts{extract}      = 1;
$opts{login}        = 1;
$opts{case}         = 1;

GetOptions(\%opts,
    'debug|d',      'help|h',     'overwrite|W',  'savebatch|T=s',
    'paste|p',      'show|s',     'delete|D',     'clear|C',
    'continue|c',   'renew|R',    'recall|r',     'format|f=s',
    'output|o=s',   'append|a=s', 'background|b', 'quiet|q',
    'grep|g=s',     'agent|U=s',  'proxy|y=s',    'savedir|S=s',
    'cclass|l=s',   'exec|x=s',   'progress|G=s', 'clivepass|V=s',
    'version|v'     => \&print_version,
    'modversion|M', => \&print_version_mods,
    # Commented out until WWW::Curl is fixed:
    # 'maxspeed!', 'minspeed!',
    # Workarounds since $longopt!|$shortopt cannot be used.
    'no-extract|n'        => sub { $opts{extract}   = 0 },
    'no-login|L'          => sub { $opts{login}     = 0 },
    'no-proxy|X'          => sub { $opts{proxy}     = "" },
    # Workaround for options with dashes. There's likely a better way.
    'ignore-case|i'       => sub { $opts{case}      = 0 },
    'filename-format|N=s' => sub { $opts{fnfmt}     = $_[1] },
    'show-format|H=s'     => sub { $opts{showfmt}   = $_[1] },
    'youtube-user|u=s',   => sub { $opts{ytuser}    = $_[1] },
    'youtube-pass|t=s',   => sub { $opts{ytpass}    = $_[1] },
    'emit-csv|e',         => sub { $opts{emitcsv}   = 1 },
    'emit-xml|E',         => sub { $opts{emitxml}   = 1 },
) or exit(1);
pod2usage(-exitstatus => 0, -verbose => 1) if $opts{help};

main();

## Subroutines: Connection

sub init_curl {
    $curl = WWW::Curl::Easy->new;

    $curl->setopt(CURLOPT_USERAGENT, $opts{agent} || "Mozilla/5.0");
    $curl->setopt(CURLOPT_FOLLOWLOCATION, 1);
    $curl->setopt(CURLOPT_AUTOREFERER, 1);
    $curl->setopt(CURLOPT_HEADER, 1);
    $curl->setopt(CURLOPT_NOBODY, 0);

    $curl->setopt(CURLOPT_VERBOSE, 1)
        if $opts{debug};

    $curl->setopt(CURLOPT_PROXY, $opts{proxy})
        if defined $opts{proxy};

    $curl->setopt(CURLOPT_MAX_RECV_SPEED_LARGE, $opts{maxspeed})
        if $opts{maxpseed}; # NOTE: No effect. Bug in WWW::Curl::Easy?

    $curl->setopt(CURLOPT_LOW_SPEED_LIMIT, $opts{minspeed})
        if $opts{minspeed}; # Ditto.
}

sub auth_youtube { # Log into Youtube
    print "[youtube] attempt to login as $opts{ytuser} ..."
        unless $opts{quiet};

    my $response = "";
    open my $fh, ">", \$response;

    my $login_url = "http://uk.youtube.com/login?current_form=loginform"
        ."&username=$opts{ytuser}&password=$opts{ytpass}"
        ."&action_login=log+in&hl=en-GB";

    $curl->setopt(CURLOPT_URL, $login_url);
    $curl->setopt(CURLOPT_COOKIEFILE, ""); # Enable cookies from here on
    $curl->setopt(CURLOPT_ENCODING, ""); # Supported encodings
    $curl->setopt(CURLOPT_WRITEDATA, $fh);

    my $rc = $curl->perform;
    my $errmsg;

    if ( $rc == 0 ) {
        $response =~ tr{\n}//d;
        $errmsg = "error: login was incorrect"
            if $response =~ /your log-in was incorrect/i;
        $errmsg = "error: check your login password"
            if $response =~ /check your password/i and !$errmsg;
        $errmsg = "error: too many login failures, try again later"
            if $response =~ /too many login failures/i and !$errmsg;
    } else {
        $errmsg = "error: ".$curl->strerror($rc)." (http/$rc)";
    }
    close $fh;

    print STDERR "\n$errmsg\n" and exit
        if $errmsg;

    print "done.\n"
        unless $opts{quiet};

    $curl->setopt(CURLOPT_COOKIE, "is_adult="
        . uc( sha1_hex(rand()) ) );

    $ytube_logged = 1;
}


# Subroutines: Queue

sub process_queue {
    init_curl();

    foreach ( @queue ) {
        $hash = sha1_hex($_);

        my $errmsg;
        my ($rc, $rfh, $response) = fetch_page($_);

        if ( $rc == 0 or $rc == 0xff ) {
            $rc = $curl->getinfo(CURLINFO_RESPONSE_CODE)
                unless $rc == 0xff; # read from cache

            if ( $rc == 200 or $rc == 0xff ) {
                if ( !defined( $entry{page_url} ) ) {
                    next if process_page($_, \$response, $rfh) == -1;
                }
                extract_video() if $entry{xurl};
            } else {
                $errmsg = $curl->strerror($rc)." (http/$rc)";
            }
        } else {
            $errmsg = $curl->strerror($rc)." (http/$rc)";
        }
        close $rfh;

        print STDERR "\nerror: $errmsg\n"
            if $errmsg;
    }
    exec_cmd();
    emit();
}

sub fetch_page {
    my ($url, $response, $from_cache, $rc) = (shift, "");
    open my $fh, ">", \$response;

    # Youtube: login only if both username and password are defined
    if ( $opts{ytuser} and $opts{ytpass} and $opts{login} ) {
        auth_youtube()
            if !$ytube_logged and $url =~ /$re_hosts{IsYoutube}/;
    }

    if ( $cache{$hash} ) {
        fetch_entry($hash); # Make sure cached "format" matches with options
        $from_cache = 1
            if $opts{format} eq $entry{file_format};
    }

    $from_cache = 0
        if $opts{renew};

    printf "%s $url ...",
        $from_cache ? "cache" : "fetch" 
            unless $opts{quiet};

    $rc = 0xff; # flag: read cache entry

    unless ( $from_cache ) {
        %entry = ();
        $curl->setopt(CURLOPT_URL, $url);
        $curl->setopt(CURLOPT_ENCODING, "");
        $curl->setopt(CURLOPT_WRITEDATA, $fh);
        $rc = $curl->perform;
    }

    return ($rc, $fh, decode_utf8($response));
}

sub process_page {
    my ($url, $response_ref, $response_fh) = @_;

    print "done.\nprocess page ..."
        unless $opts{quiet};

    $$response_ref =~ tr{\n}//d;

    my $p = HTML::TokeParser->new($response_ref);
    $p->get_tag("title");
    my $title = $p->get_trimmed_text;

    my ($xurl, $id);
    if      ( $url =~ /$re_hosts{IsYoutube}/ ) {
        ($xurl, $id) = handle_youtube($response_ref);
    } elsif ( $url =~ /$re_hosts{IsGoogle}/ ) {
        ($xurl, $id) = handle_google($response_ref);
    } elsif ( $url =~ /$re_hosts{IsSevenload}/ ) {
        ($xurl, $id, $title) = handle_sevenload($response_ref, $response_fh);
    } elsif ( $url =~ /$re_hosts{IsBreak}/ ) {
        ($xurl, $id, $title) = handle_break($response_ref);
    } elsif ( $url =~ /$re_hosts{IsLiveleak}/ ) {
        ($xurl, $id) = handle_liveleak($response_ref, $response_fh);
    } elsif ( $url =~ /$re_hosts{IsEvisor}/ ) {
        ($xurl, $id) = handle_evisor($response_ref);
    }
    return -1
        if !$xurl or !$id or !$title;

    $title =~ tr{;}//d; # Cache values cannot contain ';'

    $entry{page_url}      = $url;
    $entry{xurl}          = $xurl;
    $entry{page_title}    = $title;
    $entry{video_id}      = $id;
    $entry{file_format}   = $opts{format};

    return 0;
}

sub query_video_length {
    my ($content_type, $errmsg);

    unless ( $entry{file_length} ) {
        print "done.\nquery length ..."
            unless $opts{quiet};

        $curl->setopt(CURLOPT_URL, $entry{xurl});
        # Do not download: GET => HEAD request.
        $curl->setopt(CURLOPT_NOBODY, 1);
        my $rc = $curl->perform;
        # Reset back: HEAD => GET
        $curl->setopt(CURLOPT_HTTPGET, 1);

        $entry{file_length} =
            $curl->getinfo(CURLINFO_CONTENT_LENGTH_DOWNLOAD);

        $content_type           =
            $entry{file_suffix} =
            $curl->getinfo(CURLINFO_CONTENT_TYPE);

        $rc = $curl->getinfo(CURLINFO_RESPONSE_CODE);

        if ( $rc == 200 ) {
            my $content_ok = 0;
            if ( $content_type =~ m{video/(.*)} ) {
                $entry{file_suffix} = $1;
                if ( $content_type =~ /(.*)-(.*)$/ ) {
                    $entry{file_suffix} = $2;
                } $content_ok = 1;
            # Break, Evisor and Metacafe return "text/plain" for Content-Type
            } elsif ( $content_type =~ m{text/plain} ) {
                if ( $opts{format} eq "flv" ) {
                    if ( $entry{page_url} =~ /$re_hosts{IsBreak}/
                      or $entry{page_url} =~ /$re_hosts{IsEvisor}/ ) {
                     #or $entry{page_url} =~ /$re_hosts{IsMetacafe}/ ) {
                        $entry{file_suffix} = "flv";
                        $content_ok = 1;
                    }
                }
            # Liveleak returns "flv-application/octet-stream"
            } elsif ( $content_type =~ m{flv-application/octet-stream} ) {
                if ( $entry{page_url} =~ /$re_hosts{IsLiveleak}/ ) {
                    if ( $opts{format} eq "flv" ) {
                        $entry{file_suffix} = 'flv';
                        $content_ok = 1;
                    }
                }
            }
            $errmsg = "expected different content-type, "
                    . "received \"$content_type\""
                        unless $content_ok;
        } else {
            $errmsg = "server returned http/$rc";
        }
    } else { # Construct content-type from cache
        $content_type = "video/$entry{file_suffix}";
    }

    unless ( $opts{quiet} ) {
        if ( !$errmsg ) { print "done.\n"; } 
        else { print STDERR "\nerror: $errmsg\n"; }
    }

    return ($errmsg ? -1:0, $content_type);
}

sub extract_video {
    my ($rc, $content_type) = query_video_length();

    return
        if $rc != 0 or !defined $content_type;

    my $fn          = title_to_filename($entry{page_title});
    my $path        = File::Spec->catfile( $opts{savedir} || $workdir, $fn );
    my $filemode    = ">";
    my $remaining   = $entry{file_length};
    my $size        = -s $path || 0;
    my $cont_from   = 0;
    my $errmsg;

    save_entry($hash);

    if ( $size > 0 and !$opts{overwrite} ) {
        if ( $size == $entry{file_length} and $opts{extract} ) {
            print "ignored: localfile_length == remotefile_length. "
                . "refusing to proceed any further.\n";

            push @exec_files,$path
                if $opts{exec};

            return
                unless $opts{emitcsv} or $opts{emitxml};

        } elsif ( $size < $entry{file_length} and $opts{continue} ) {
            $cont_from  = $size;
            $filemode   = ">>";
            $remaining  = ($entry{file_length} - $cont_from);
        } else {
            ($path,$fn) =
                newname_if_exists( $opts{savedir} || $workdir, $fn );
        }
    }

    if ( $opts{emitcsv} or $opts{emitxml} ) {
        $entry{fn}          = $fn;
        $entry{remaining}   = $remaining;
        $entry{cont_from}   = $cont_from;
        push @emit_queue, {%entry};
        return;
    }

    unless ( $opts{quiet} ) {
        print "file: $fn\n";
        print "length: $entry{file_length} ";

        printf"(%.2fMB)  ",$entry{file_length}/MBDIV
            if $entry{file_length};

        printf "from: %u (left: %u)  ", $cont_from, $remaining
            if $cont_from;

        printf "[$content_type]"
            if $content_type;

        print "\n";
    }

    if ( $rc == 0 ) {
        return
            unless $opts{extract};

        if ( open my $fh, "$filemode$path" ) {
            # Disable: encoding, header
            $curl->setopt(CURLOPT_HEADER, 0);
            $curl->setopt(CURLOPT_ENCODING, "identity");
            $curl->setopt(CURLOPT_URL, $entry{xurl});
            $curl->setopt(CURLOPT_WRITEDATA, $fh);

            $curl->setopt(CURLOPT_RESUME_FROM, $cont_from)
                if $cont_from;

            unless ( $opts{quiet} ) {
                $curl->setopt(CURLOPT_PROGRESSFUNCTION, \&progress_callback);
                $curl->setopt(CURLOPT_NOPROGRESS, 0);
                $time_started   = time;

                # Use 'dot' progress if the output is not a TTY
                if ( $opts{progress} !~ /^dot/ and $opts{progress} ne 'none' ) {
                    $opts{progress} = 'dot'
                        if ! -t STDOUT or ! -t STDERR;
                }

                if ( $opts{progress} =~ /^bar/ ) {
                    bar_init($cont_from, $entry{file_length});
                } elsif ( $opts{progress} =~ /^dot/ ) {
                    dot_init()
                }
            }

            $rc = $curl->perform;
            close $fh;

            if ( $rc == 0 ) {
                $rc = $curl->getinfo(CURLINFO_RESPONSE_CODE);
                if ( $rc == 200 or $rc == 206 ) {
                    if    ( $opts{progress} =~ /^bar/ ) { bar_finish() }
                    elsif ( $opts{progress} =~ /^dot/ ) { dot_finish() }
                } else {
                    $errmsg = $curl->strerror($rc)." (http/$rc)";
                }
            }
            else {
                $errmsg = $curl->strerror($rc)." (http/$rc)";
            }

            # Reset
            $curl->setopt(CURLOPT_RESUME_FROM, 0);
            $curl->setopt(CURLOPT_HEADER, 1);
        } else {
            $errmsg = "$path: $!";
        }
    } else {
        $errmsg = $curl->strerror($rc)." (http/$rc)";
    }

    if ( ! $errmsg ) {
        print "\nclosed http/$rc.\n"
            unless $opts{quiet};

        push @exec_files,$path
            if $opts{exec};
    } else {
        print STDERR "\nerror: $errmsg\n";
    }

    # Disable: progress
    $curl->setopt(CURLOPT_NOPROGRESS, 1);
}

sub get_queue {
    if ( $opts{recall} and -e $RECALLFILE ) {
        if ( open my $fh, "<$RECALLFILE" ) {
            parse_input($_) while ( <$fh> );
            close $fh;
        } else {
            print STDERR "error: $RECALLFILE: $!";
        }
    }

    if ( $opts{paste} ) {
        print STDERR "error: Clipboard module not found\n" and exit
            unless $opted_mods{Clipboard};
        my $data = Clipboard->paste();            
        if ( $data ) {
            parse_input($_) foreach split(/\n/,$data);
        }
    }

    parse_input($_) foreach @ARGV;
    grep_cache() if $opts{grep};
    unless ( @queue ) { parse_input($_) while ( <STDIN> ); }

    my %h = map { $_, 1 } @queue; # Remove duplicates from the URL queue
    @queue = keys %h;

    if ( open my $fh, ">$RECALLFILE" ) {
        print $fh "$_\n" foreach @queue;
        close $fh;
    } else {
        print STDERR "error: $RECALLFILE: $!";
    }

    if ( $opts{savebatch} ) {
        if ( open my $fh, ">", $opts{savebatch} ) {
            print $fh "$_\n" foreach @queue;
            close $fh;
        } else {
            print STDERR "error: $opts{savebatch}: $!";
        }
    }
}

sub parse_input {
    my $url = shift;

    return if $url =~ /^$/;
    return if $url =~ /^#/;

    chomp $url;

    if ( $url =~ /&srcurl=(.*?)&/ ) { # GVideo: one of many redirects
        printf "found redirect ...%s\n=> %s\n",
            (split(/&/,$url))[0],
            (split(/&/,uri_unescape($1)))[0]
                unless $opts{quiet};
        $url = uri_unescape($1);
    }

    # Insert http:// if not found
    $url = "http://$url"
        if $url !~ m{^http://}i;

    # Translate embedded URL to video page URL
    translate_embed(\$url);

    # Last.fm wraps Youtube videos as their own
    if ( $url =~ /$re_hosts{IsLastfm}/ ) {
        $url =~ /\Q+1-\E(.+)/;

        print STDERR "error: nosupport: $url\n" and return -1
            unless defined($1);

        $url = "http://youtube.com/watch?v=$1";
    }

    # Remove params from the URL
    ($url) = split(/&/,$url);

    foreach my $re ( %re_hosts ) {
        push @queue,$url and return 0
            if $url =~ /$re/;
    }

    print STDERR "error: nosupport: $url\n";

    return -1;
}


# Subroutines: Video page handlers

sub handle_youtube {
    my ($response_ref, $xurl) = (shift);

    my %re = (
        GrabID  => qr/"video_id": "(.*?)"/,
        GrabT   => qr/"t": "(.*?)"/
    );

    my $id = $1 if $$response_ref =~ /$re{GrabID}/;
    my $t  = $1 if $$response_ref =~ /$re{GrabT}/;

    if ( $id and $t ) {
        $xurl = "http://youtube.com/get_video?video_id=$id&t=$t";

        my $fmt;
        if      ( $opts{format} eq  "mp4" ) { $fmt = 18; }
        elsif   ( $opts{format} eq "3gpp" ) { $fmt = 17; }
        elsif   ( $opts{format} eq "xflv" ) { $fmt =  6; }

        $xurl .= "&fmt=$fmt"
            if $fmt;
    } else {
        printf STDERR "\nerror: failed to extract &%s\n",
            $id ? "t"
                : "video_id";
    }
    return ($xurl, $id);
}

sub handle_google {
    my $response_ref = shift;

    my %re = (
        #GrabRedirect => qr|lfRedirect\('(.*?)'|,
        GrabVideoURL => qr|\Qgoogleplayer.swf?videoUrl\x3d\E(.*?)\Q\x26|,
        GrabID       => qr|docid: '(.*?)'|,
        GrabMP4      => qr|\Qhref="http://vp.\E(.*?)"|,
    );

    #my $redir  = $1 if $$response_ref =~ /$re{GrabRedirect}/;
    my $xurl   = uri_unescape($1)
        if $$response_ref =~ /$re{GrabVideoURL}/;

    my $id     = $1 if $$response_ref =~ /$re{GrabID}/;
    my $mp4    = $1 if $$response_ref =~ /$re{GrabMP4}/;

#    if ( $redir ) {
# video.google.* http/302 redirects to the actual video hosts again.
# Leaving this commented out until they decide to flip it back on
# again after some mind-boggling brainstorming.
#        $redir =~ s{\\x3d}{=};
#        push @queue, $redir;
#        print "Found a redirect to another host. Pushed into queue.\n"
#            unless $opts{quiet};
#    } else {
        $xurl = "http://vp.$mp4"
            if $mp4 and $opts{format} eq "mp4";

        print STDERR "\nerror: extraction url not found\n"
            unless $xurl;
#    }
    return ($xurl, $id);
}

sub handle_sevenload {
    my ($response_ref, $response_fh) = @_;

    my %re       = ( GrabConfigPath => qr|configPath=(.*?)"| );

    my $confpath = uri_unescape($1)
        if $$response_ref =~ /$re{GrabConfigPath}/;

    my ($id, $xurl, $title);
    if ( $confpath ) {
        ($xurl, $id, $title) =
            fetch_sevenload_configxml($confpath, $response_fh);
    } else {
        print STDERR "\nerror: configPath not found\n";
    }
    return ($xurl, $id, $title);
}

sub handle_break {
    my $response_ref = shift;

    my %re = (
        GrabTitle    => qr|id="vid_title" content="(.*?)"|,
        GrabID       => qr|ContentID='(.*?)'|,
        GrabFilePath => qr|ContentFilePath='(.*?)'|,
        GrabFileName => qr|FileName='(.*?)'|
    );

    my $title   = $1 if $$response_ref =~ /$re{GrabTitle}/;
    my $id      = $1 if $$response_ref =~ /$re{GrabID}/;
    my $fpath   = $1 if $$response_ref =~ /$re{GrabFilePath}/;
    my $fname   = $1 if $$response_ref =~ /$re{GrabFileName}/;

    my ($xurl, $errmsg);
    if ( $fpath and $fname ) {
        $xurl = "http://media1.break.com/dnet/media/$fpath/$fname.flv";
    } else {
        $errmsg = "failed to extract ContentFilePath"
            if !$fpath;

        $errmsg = "failed to extract FileName"
            if !$fname and !$errmsg;
    }

    $errmsg = "failed to extract title"
        if !$title and !$errmsg;

    $errmsg = "failed to extract id"
        if !$id and !$errmsg;

    print STDERR "\nerror: $errmsg\n"
        if $errmsg;

    return ($xurl, $id, $title);
}

sub handle_liveleak {
    my ($response_ref, $response_fh) = @_;

    my %re = (
        GrabID          => qr|token=(.*?)&|,
        GrabConfigURL   => qr|'config','(.*?)'|,
    );

    my $id = $1
        if $$response_ref =~ /$re{GrabID}/;

    my $conf_url = uri_unescape($1)
        if $$response_ref =~ /$re{GrabConfigURL}/;

    my $xurl;
    if ( $conf_url ) {
        $xurl = fetch_liveleak_config($conf_url);
        # Re-enable: header, reset WRITEDATA, the above overrides the
        # original settings.
        $curl->setopt(CURLOPT_HEADER, 0);
        $curl->setopt(CURLOPT_WRITEDATA, $response_fh);
    } else {
        print STDERR "error: config url not found\n";
    }
    return ($xurl, $id);
}

sub handle_evisor {
    my $respr = shift;

    my %re = (
        GrabXurl    => qr|file=(.*?)"|,
        GrabID      => qr|.+/(.*?).flv|,
    );

    my ($xurl, $id, $errmsg);

    $xurl = $1
        if $$respr =~ /$re{GrabXurl}/;

    $id = $1
        if $xurl and $xurl =~ /$re{GrabID}/;

    $errmsg = "video extraction url not found"
        unless $xurl;

    $errmsg = "video id not found"
        unless $id and !$errmsg;

    print STDERR "error: $errmsg\n"
        if $errmsg;

    return ($xurl, $id);
}


# Subroutines: Progress
# NOTE: the 'dot' progress copies much from wget.

sub progress_callback {
    if    ( $opts{progress} =~ /^dot/ ) { dot_update(@_); }
    elsif ( $opts{progress} =~ /^bar/ ) { bar_update(@_); }
    return 0;
}

sub dot_init {
    $dp{dots}     = 0;
    $dp{rows}     = 0;
    $dp{dlthen}   = 0;
    $dp{accum}    = 0;

    # Default style
    $dp{dot_bytes}    = 1024;
    $dp{dot_spacing}  = 10;
    $dp{dots_in_line} = 50;

    my ($type,$style) = split(/:/,$opts{progress});

    if ( $style ) {
        if ( $style eq 'binary' ) {
            $dp{dot_bytes}    = 8192;
            $dp{dot_spacing}  = 16;
            $dp{dots_in_line} = 48;
        } elsif ( $style eq 'mega' ) {
            $dp{dot_bytes}    = 65536;
            $dp{dot_spacing}  = 8;
            $dp{dots_in_line} = 48;
        }
    }
}

sub dot_update {
    my ($clientp, $dltotal, $dlnow, $ultotal, $ulnow) = @_;

    my ($percent, $elapsed, $rate, $eta) =
        calc_progress($dlnow, $dltotal, 1);

    return 0
        if $elapsed < 1.0;

    my $row_bytes   = $dp{dot_bytes} * $dp{dots_in_line};

    $dp{accum}   += $dlnow - $dp{dlthen};
    $dp{dlthen}   = $dlnow;

    for (; $dp{accum} >= $dp{dot_bytes}; $dp{accum} -= $dp{dot_bytes}) {

        printf "\n%6dK", $dp{rows} * $row_bytes / 1024
            if $dp{dots} == 0;

        print " "
            if $dp{dots} % $dp{dot_spacing} == 0;

        ++$dp{dots};
        print ".";

        if ( $dp{dots} >= $dp{dots_in_line} ) {
            ++$dp{rows};
            $dp{dots} = 0;

            dot_print_row_stats($percent, $elapsed, $eta, $rate, 0);
        }
    }
}

sub dot_finish {
    return if $opts{quiet};

    my $row_bytes = $dp{dot_bytes} * $dp{dots_in_line};

    printf "\n%6dK", $dp{rows} * $row_bytes / 1024
        if $dp{dots} == 0;

    for (my $i=$dp{dots}; $i<$dp{dots_in_line}; $i++) {
        print " "
            if $i % $dp{dot_spacing} == 0;

        print " ";
    }

    my $elapsed = time - $time_started;
    my $eta     = time2str($elapsed, 1);
    my $rate    = $entry{file_length} / $elapsed;

    dot_print_row_stats(100, $elapsed, $eta, $rate, 1);
}

sub dot_print_row_stats {
    my ($percent, $elapsed, $eta, $rate, $last) = @_;
    my %rate = normalize_rate($rate);

    printf "%3d%% %4.1f%s", $percent, $rate{rate}, $rate{units};
    printf "%s%s", $last ? "=":" ", $eta;
}

sub bar_init { # A lazy man's progressbar
    my ($initial, $total) = @_;

    $total = $initial
        if $initial > $total;

    $bp{initial} = $initial; # bytes dl previously
    $bp{total}   = $total;   # expected bytes

    $bp{last_update} = 0;

    bar_update(-1, $total, 0);
}

sub bar_update { # See no bar, hear no bar, speak no bar.
    my ($clientp, $dltotal, $dlnow, $ultotal, $ulnow, $done) = @_;

    my $elapsed = time - $time_started;

    if ( !$done ) {
        return
            if ( $elapsed - $bp{last_update} < BP_REFRESH_INTERVAL);
    } else {
        $dlnow = $bp{total};
    }

    $bp{last_update} = $elapsed;

    my $size = $bp{initial} + $dlnow;
    my $bar = "";

    if ( $bp{total} > 0 ) {
        my $percent = 100.0 * $size / $bp{total};
        if ( $percent < 100 ) {
            $bar .= sprintf("%2d%% ",$percent);
        } else {
            $bar .= "100%";
        }
    }

    if ( !$done ) {
        my ($percent_, $elapsed_, $rate, $eta) = 
            calc_progress($dlnow, $dltotal, 1);

        my %rate = normalize_rate($rate);

        $bar .= sprintf(" | %8.2fMB ", $size/MBDIV);
        $bar .= sprintf(" | %8.2f%s/s",$rate{rate},$rate{units});
        $bar .= sprintf(" | ETA %-8s", $eta);
    } else {
        $bar .= sprintf(" | %6.2fMB ",$dlnow/MBDIV);
        $bar .= sprintf(" | %8.2f%s/s",0,"K");
        $bar .= sprintf(" | ETA %-8s","0s");
    }

    $bp{count} = $dlnow;
    print "\r".$bar;
}

sub bar_finish { # Never again.
    return if $opts{quiet};

    if ( $bp{total} > 0
         && $bp{count} + $bp{initial} > $bp{total} ) {
        $bp{total} = $bp{initial} + $bp{count};
    }

    bar_update(-1,-1,-1,-1,-1, 1);
}

sub calc_progress {
    my ($dlnow, $dltotal, $condensed, $elapsed) = @_;

    my $percent = 0;

    $percent = int ($dlnow / $dltotal * 100)
        if $dltotal;

    $elapsed = time - $time_started
        unless $elapsed;

    my $eta     = '--:--';
    my $rate    = 0;

    $rate = $dlnow / $elapsed
        if $elapsed;

    if ( $rate > 0 ) {
        my $left = ( $dltotal - $dlnow ) / $rate;
        $eta     = time2str($left, $condensed);
    }

    return ($percent, $elapsed, $rate, $eta);
}

sub time2str {
    my ($secs, $condensed) = @_;

    my $space =
        $condensed ? ""
                   : " ";

    my $str;
    if ( $secs < 100 ) {
        $str = sprintf("%ds", $secs);
    } elsif ( $secs < 100 * 60 ) {
        $str = sprintf("%dm%s%ds", $secs / 60, $space, $secs % 60);
    } elsif ( $secs < 48 * 3600) {
        $str = sprintf("%dh%s%dm", $secs / 3600, $space, ($secs / 60) % 60);
    } elsif ( $secs < 100 * 86400) {
        $str = sprintf("%dd%s%dh", $secs / 86400, $space, ($secs / 3600) % 60);
    } else {
        $str = sprintf("%dd", $secs / 86400);
    }
    return $str;
}

sub normalize_rate {
    my $rate = shift;

    my @names = qw/KB MB GB/;
    my $units;

    if ( $rate < 1024*1024 ) {
        $units = 0;
        $rate /= 1024;
    } elsif ( $rate < 1024*1024 ) {
        $units = 1;
        $rate /= 1024*1024;
    } else {
        $units = 2;
        $rate /= 1024*1024*1024;
    }
    return (rate => $rate, units => $names[$units]);
}


# Subroutines: LittleHelpers

sub main {
    init_cache();

    if      ( $opts{clear} ) { clear_cache(); }
    elsif   ( $opts{show} )  { show_cache(); }

    verify_exec();

    grab_clivepass();
    get_queue();

    select STDERR; $| = 1; # => unbuffered
    select STDOUT; $| = 1;

    daemonize()
        if $opts{background};

    process_queue();

    free_cache();
}

sub grab_clivepass {
    # TODO: Supports only Youtube. Expand to support other websites as needed.
    return
        unless $opts{login} and $opts{ytuser} and $opts{ytpass} eq "-";

    print STDERR "error: no path to clivepass, use --clivepass\n" and exit
        unless $opts{clivepass};

    print STDERR "error: Expect module not found\n" and exit
        unless $opted_mods{Expect};

    my $phrase;
    $phrase = getpass("Enter passphrase for clivepass: ")
        while ( ! $phrase );

    my $e = Expect->new;
    $e->log_stdout(0);
    $e->spawn($opts{clivepass}, "-g", $opts{ytuser})
        or print STDERR "error: could not spawn: $!\n" and exit;

    my ($spawned, $pwd);
    $e->expect(10, [
        qr'Enter passphrase: $',
        sub {
            my $fh = shift;
            $fh->send("$phrase\n");
            $spawned = 1;
            exp_continue;
        }
    ],
    [ eof => sub {
        if ( $spawned ) {
            my $fh = shift;
            $pwd = $fh->before();
            if ( $pwd =~ /error: (.*?)$/ ) {
                print STDERR "clivepass: error: $1\n";
                exit;
            } else {
                $pwd = $1 if ( $pwd =~ /login: $opts{ytuser}=(.*?)$/ );
            }
        } else {
            print STDERR "error: could not spawn $opts{clivepass}\n";
            exit;
        }
    }],
    [ timeout => sub {
        print STDERR "error: clivepass: expect timed out\n";
        exit;
    }]);

    $opts{ytpass} = $pwd;
}

sub getpass {
    system "stty -echo";
    print shift;
    chomp(my $pwd = <STDIN>);
    print "\n";
    system "stty echo";
    return $pwd;
}

sub daemonize {
    $logfile = $opts{append}
        || $opts{output}
        || File::Spec->catfile( $workdir, "clive-log" );

    my $pid = fork;
    if ( $pid < 0 ) {
        print STDERR "\nerror: fork failed: $!";
        exit 1;
    } elsif ( $pid != 0 ) {
        print "continuing in background, pid $pid.\n";
        print "output will be written to $logfile.\n"
            unless $opts{quiet};
        exit 0;
    }

    chdir $workdir;

    my $mode = $opts{append} ? ">>" : ">";
    $logfile = "/dev/null" if $opts{quiet};

    open STDOUT, "$mode", "$logfile"
        or die "error: cannot redirect STDOUT: $!";

    open STDERR, ">&STDOUT"
        or die "error: cannot dup STDOUT: $!";
}

sub fetch_liveleak_playlist {
    my $playlist_url = shift;

    print "done.\nfetch playlist xspf ..."
        unless $opts{quiet};

    my $playlist = "";
    open my $fh, ">", \$playlist;

    $curl->setopt(CURLOPT_URL, $playlist_url);
    $curl->setopt(CURLOPT_WRITEDATA, $fh);

    my $rc = $curl->perform;
    close $fh;

    my $xurl;
    if ( $rc ==  0 ) {
        # NOTE: XML::XSPF exists in CPAN but this should work just as well.
        # Parsing with XML::Simple results in errors due unescaped values.
        $playlist =~ tr{\n}//d;
        my $re = qr|<location>(.*?)</location>|;
        $xurl = $1 if $playlist =~ /$re/;
    } else {
        print STDERR "\nerror: " . $curl->strerror($rc) . " (http/$rc)\n";
    }

    print STDERR "\nerror: track <location> not found\n"
        unless $xurl;

    return $xurl;
}

sub fetch_liveleak_config {
    my $config_url = shift;

    print "done.\nfetch config xml ..."
        unless $opts{quiet};

    my $config = "";
    open my $fh, ">", \$config;

    # Disable: header
    $curl->setopt(CURLOPT_HEADER, 0);
    $curl->setopt(CURLOPT_URL, $config_url);
    $curl->setopt(CURLOPT_WRITEDATA, $fh);

    my $rc = $curl->perform;
    close $fh;

    my $xurl;
    if ( $rc == 0 ) {
        my $xml = XMLin($config);
        #use Data::Dumper; print Dumper($xml);
        $xurl = fetch_liveleak_playlist($xml->{file});
    } else {
        print STDERR "\nerror: " . $curl->strerror($rc) . " (http/$rc)\n";
    }

    return $xurl;
}

sub fetch_sevenload_configxml {
    my ($conf_url, $response_fh) = @_;

    print "done.\nfetch config xml..."
        unless $opts{quiet};

    my $conf_xml = "";
    open my $conf_fh, ">", \$conf_xml;

    # Disable: header
    $curl->setopt(CURLOPT_HEADER, 0); 
    $curl->setopt(CURLOPT_URL, $conf_url);
    $curl->setopt(CURLOPT_WRITEDATA, $conf_fh);

    my $rc = $curl->perform;
    close $conf_fh;

    # Re-enable: header
    $curl->setopt(CURLOPT_HEADER, 1);
    $curl->setopt(CURLOPT_WRITEDATA, $response_fh);

    my ($id, $xurl, $title);

    if ( $rc == 0 ) {
        my $xml = XMLin($conf_xml);
        #use Data::Dumper; print Dumper($xml);
        $title  = $xml->{playlists}{playlist}{items}{item}{title};
        $id     = $xml->{playlists}{playlist}{items}{item}{id};
        $xurl   = $xml->{playlists}{playlist}{items}{item} # Monstrous.
                    {videos}{video}{streams}{stream}{locations}{location}{content};
    } else {
        print STDERR "\nerror: " . $curl->strerror($rc) . " (http/$rc)\n";
    }

    my $errmsg;

    $errmsg = "failed to extract item title"
        if !$title;

    $errmsg = "failed to extract item id"
        if !$id and !$errmsg;

    print STDERR "\nerror: $errmsg\n"
        if $errmsg;

    return ($xurl, $id, $title);
}

sub title_to_filename {
    my $title = shift;

    $title =~ s/youtube - //i;  # Remove host specific strings from title
    $title =~ s/ video//i;      # Break
    $title =~ s/\Qliveleak.com - \E//i;

    my $r = $opts{cclass} || qr|\w|;
    $title = join('',$title =~ /$r/g);

    # Courtesy of:
    #   http://search.cpan.org/~gaas/URI-1.37/URI.pm#PARSING_URIs_WITH_REGEXP
    my ($scheme, $authority, $path, $query, $fragment) =
        m{(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?}o;
    # Extract the domain from the URL.
    my @a = split(/\./,$authority);

    my $fn          = $opts{fnfmt} || "%t-(%i)-[%d].%s";
    my $timestamp   = strftime("%F %T",localtime);

    my %h = (
        "%t" => $title,
        "%s" => $entry{file_suffix},
        "%d" => $a[scalar @a-2], # Without the TLD.
        "%i" => $entry{video_id},
        "%D" => (split(/ /,$timestamp))[0],
        "%T" => (split(/ /,$timestamp))[1],
        "%S" => $timestamp,
    );

    my $m = join('|',keys %h);
    $fn =~ s/($m)/$h{$1}/ig;

    return $fn;
}

sub newname_if_exists {
    my ($path, $orig, $new) = (shift, shift);

    for ( my $i=1;; $i++ ) {
        $new = File::Spec->catfile( $path, "$orig.$i" );
        last if ! -e $new;
    }

    my ($vol, $dir, $fn) = File::Spec->splitpath($new);
    return ($new, $fn);
}

sub format_show {
    my $s   = shift;
    my %e   = map_entry(shift);

    my $t = $opted_mods{IOPager}
        ? $e{page_title}
        : decode_utf8($e{page_title});

    my %h   = (
        "%t" => $t,
        "%i" => $e{video_id},
        "%l" => $e{file_length},
        "%m" => sprintf("%.2f", $e{file_length}/MBDIV),
        "%u" => $e{page_url},
        "%x" => $e{xurl},
        "%D" => (split(/ /,$e{time_stamp}))[0],
        "%T" => (split(/ /,$e{time_stamp}))[1],
        "%S" => $e{time_stamp},
    );

    my $m = join('|',keys %h);
    $s =~ s/($m)/$h{$1}/ig;

    return $s;
}

sub init_cache {
    mkpath( [$CONFIGDIR], 0, 0700 );
    $cache_db = tie %cache, "BerkeleyDB::Hash",
        -Filename => $CACHEFILE,
        -Flags => DB_CREATE
        or die "error: cannot open $CACHEFILE: $! $BerkeleyDB::Error\n";
}

sub show_cache {
    IO::Pager->new(*STDOUT)
        if $opted_mods{IOPager};

    my $fmt = $opts{showfmt} || SHOWFMT_DEFAULT;
    my @entries = ();

    if ( $opts{grep} ) {
        grep_cache(); # Stores matches => @queue
        push @entries, format_show( $fmt, sha1_hex($_) )
            foreach ( @queue );
    } else {
        push @entries, format_show( $fmt, $_ )
            foreach ( sort keys %cache );
    }

    print STDOUT "$_\n"
        foreach sort @entries;

    close STDOUT
        if $opted_mods{IOPager};

    if ( $opts{grep} and $opts{delete} and scalar @queue > 0 ) {
        print "Confirm delete (y/N):";
        $_ = lc <STDIN>;
        chomp;
        if ( lc $_ eq "y" ) {
            delete $cache{sha1_hex($_)}
                foreach ( @queue );
        }
    }
    exit;
}

sub clear_cache {
    unlink $CACHEFILE if -e $CACHEFILE;
    exit;
}

sub free_cache {
    undef $cache_db;
    untie %cache;
}

sub map_entry {
    my $key     = shift;
    my @values  = split(/;/,$cache{$key});

    my @keys = qw(
        file_suffix file_length file_format page_title
        page_url    time_stamp  video_id    xurl
    ); # Order matters. See also save_entry.

    my $i = 0;
    return map { $_ => $values[$i++] } @keys;
}

sub fetch_entry {
    %entry = map_entry($hash);
    $entry{page_title} = decode_utf8($entry{page_title});
    #while (my ($key, $value) = each(%entry)) { print "$key => $value\n"; } die;
}

sub save_entry {
    my @values;

    $entry{time_stamp} = strftime("%F %T",localtime);

    push @values,$entry{$_}
        foreach sort keys %entry;

    $cache{$hash} = join(';',@values);
    $cache_db->db_sync();
}

sub grep_cache {
    my $g   =
        $opts{case} ? qr|$opts{grep}|
                    : qr|$opts{grep}|i;

    my $fmt = $opts{showfmt} || SHOWFMT_DEFAULT;

    foreach ( sort keys %cache ) {
        my @e = split(/;/,$cache{$_});
        if ( grep /$g/, @e ) {
            if   ( $opts{delete} ) {
                if ( $opts{show} ) { push @queue,$e[4]; }
                else { delete $cache{$_}; }
            }
            else { push @queue,$e[4]; } # 4=URL
        }
    }
    exit
        if $opts{delete} and not $opts{show};
}

sub translate_embed {
    my ($url) = @_;
    $$url =~ s!/v/!/watch?v=!i;              # youtube
    $$url =~ s!googleplayer.swf!videoplay!i; # googlevideo
    $$url =~ s!/pl/!/videos/!i;              # sevenload
    $$url =~ s!/e/!/view?i=!i;               # liveleak
}

sub verify_exec {
    return if !$opts{exec};
    if ($opts{exec} !~ /[;+]$/) {
        print "error: --exec expression must be terminated "
            . "by either ';' or '+'\n";
        exit;
    }
}

sub exec_cmd {
    return if !$opts{exec};
    if ($opts{exec} =~ /;$/) { # semi
        foreach (@exec_files) {
            my $cmd = sprintf('"%s" ',$opts{exec});
            $cmd =~ s/%i/"$_"/g;
            $cmd =~ tr{;}//d;
            system("$cmd");
        }
    } else { # plus
        my $cmd = sprintf("%s ",$opts{exec});
        $cmd =~ tr{%i}//d;
        $cmd =~ tr{+}//d;
        $cmd .= sprintf('"%s" ',$_)
            foreach (@exec_files);
        system("$cmd");
    }
}

sub emit {
    print "<?xml version=\"1.0\"?>\n<queue>\n"
        if $opts{emitxml} and @emit_queue;

    foreach ( @emit_queue ) {
        if ( $opts{emitxml} ) {
            print "  <video>\n";
            while ( my ($key,$value) = each (%$_) ) {
                $value = uri_escape($value)
                    if $key eq 'xurl' or $key eq 'page_url';
                print "    <$key>$value</$key>\n";
            }
            print "  </video>\n";
        } elsif ( $opts{emitcsv} ) {
            printf qq/csv:"%s","%s","%s","%.2fMB",/
                 . qq/"%s","%s","%s","%s","%s","%s"\n/,
                $_->{page_url}, $_->{xurl}, $_->{fn},
                $_->{file_length}/MBDIV, $_->{file_length},
                $_->{video_id}, $_->{time_stamp}, $_->{page_title}, 
                $_->{cont_from}, $_->{remaining};
        }
    }
    print "</queue>\n"
        if $opts{emitxml} and @emit_queue;
}

sub print_version {
    print "clive ".VERSION.". Copyright (c) 2007-2009 Toni Gundogdu.\n"
        . "clive is licensed under the ISC license which is functionally\n"
        . "equivalent to the 2-clause BSD licence.\n";
    exit;
}

sub print_version_mods {
    my $perl_v      = sprintf "%vd", $^V;
    my $clipboard_v = $opted_mods{Clipboard} ? $Clipboard::VERSION  : "-";
    my $expect_v    = $opted_mods{Expect}    ? $Expect::VERSION     : "-";
    my $iopager_v   = $opted_mods{IOPager}   ? $IO::Pager::VERSION  : "-";
print
" * Perl/$perl_v
Modules:
 * Config::Tiny/$Config::Tiny::VERSION\t\t* BerkeleyDB/$BerkeleyDB::VERSION
 * WWW::Curl/$WWW::Curl::VERSION\t\t* Expect/$expect_v
 * HTML::TokeParser/$HTML::TokeParser::VERSION\t* Clipboard/$clipboard_v
 * XML::Simple/$XML::Simple::VERSION\t\t* IO::Pager/$iopager_v
 * URI::Escape/$URI::Escape::VERSION
Core modules:
 * POSIX/$POSIX::VERSION\t\t\t* Cwd/$Cwd::VERSION
 * Getopt::Long/$Getopt::Long::VERSION\t\t* Pod::Usage/$Pod::Usage::VERSION
 * File::Path/$File::Path::VERSION\t\t* File::Spec/$File::Spec::VERSION
 * Digest::SHA/$Digest::SHA::VERSION\t\t* Encode/$Encode::VERSION
 * File::Basename/$File::Basename::VERSION
"; exit;
}

__END__

=head1 SYNOPSIS

clive [options]... [URL]...

=head1 OPTIONS

 -h, --help                 print help and exit
 -v, --version              print version and exit
 -M, --modversion           print perl and module versions and exit
 -b, --background           go to background after startup
 -e, --emit-csv             emit video details as csv to stdout
 -E, --emit-xml             emit video details as csv to stdout
 -U, --agent=STRING         identify as STRING to http server
 -y, --proxy=ADDR           use ADDR for http proxy
 -X, --no-proxy             do not use http proxy
 -R, --renew                renew cache entry for visited url
 -s, --show                 dump cache entries to stdout
 -H, --show-format=STRING   format dumped cache entries
 -g, --grep=PATTERN         grep cache entries for PATTERN
 -i, --ignore-case          ignore case-differences with --grep
 -D, --delete               delete matched entries from cache
 -C, --clear                clear cache of all entries
 -o, --output=LOGFILE       log messages to LOGFILE
 -a, --append=LOGFILE       append to LOGFILE
 -d, --debug                print libcurl debug messages
 -q, --quiet                turn off all output
 -r, --recall               recall last url batch
 -T, --savebatch=FILE       save url batch to FILE
 -p, --paste                paste input from clipboard
 -n, --no-extract           do not extract any videos
 -c, --continue             continue partially downloaded file
 -W, --overwrite            overwrite existing video file
 -G, --progress=TYPE        use progress indicator TYPE
 -u, --youtube-user=UNAME   youtube username
 -t, --youtube-pass=PASSW   youtube password
 -L, --no-login             do not log into youtube
 -S, --savedir=DIR          save video files to DIR
 -f, --format=FORMAT        extract video FORMAT
 -l, --cclass=CLASS         use CLASS to filter titles
 -N, --filename-format=STR  use STR to construct output filename
 -x, --exec=COMMAND         execute COMMAND subsequently
 -V, --clivepass=PATH       path to clivepass
