#!/bin/bash
#
# Primary (Master) Server
# Failover functions for ENBD + Fast RAID-1
#
# Author:	Steve Purkis <steve.purkis@multimap.com>
#		(derived work)
# Originals by:	Peter T. Breuer <ptb@it.uc3m.es>
#		Matthew Sackman <matthew@wellquite.org>
# License:	GNU GPL v2.1
#
# These scripts are for Red Hat machines.
# Designed to be used with Linux-HA (aka Heartbeat).

# Source RedHat function library.  
. /etc/init.d/functions

NBD="/dev/nda"
LOCAL_DEVICE="/dev/hda10"
RAID_DEVICE="/dev/md0"
FS_TYPE="ext2"
MOUNTPOINT="/usr/local/perforce"
MOUNT_OPTS=""
SERVER_FAILED_FILE="/tmp/enbd-primary"

# How to tell if we're reviving the master, or starting from scratch?
# Options:
#    1. echo our status to a file?
#    2. put a timestamp on each device in raid array, and check
#    3. write a little daemon that is kicked-off if the secondary is acting
#       as master.  If you can connect to it from primary then revive.
start_or_revive_primary() {
	is_primary_already_running
	if [ $? -eq 0 ]; then
		echo "Primary is already running, not attempting to start"
		return
	fi

	if [ -e $SERVER_FAILED_FILE ]; then
		echo "$SERVER_FAILED_FILE exists: deciding whetther to revive or start"
		service enbd-client status
		ALIVE=$?
		if [ $ALIVE -ne 0 ]; then
			echo "NOTE: reviving primary server"
			start_primary_after_failure
			# update timestamp
			touch $SERVER_FAILED_FILE
		else
			start_primary
		fi
	else
		start_primary
		touch $SERVER_FAILED_FILE
	fi
}

is_primary_already_running() {
    # Tough one... primary is up if:
    #   * outside world is pingable
    #   * raid device is mounted
    #   * NBD in /proc/mdstat has failed
    #
    is_device_mounted $RAID_DEVICE
    ALREADY_MOUNTED=$?
    if [ $ALREADY_MOUNTED -eq 0 ]; then
    	echo "WARNING: RAID device $RAID_DEVICE is already mounted"
	echo "         assuming primary is already running!"
    	return 0
    else
    	return 1
    fi
}

# Scenario 1 - Normal operation, startup
start_primary() {
    # TODO: handle the case where secondary is not available!
    #       (or is this done by heartbeat?)
    service enbd-client start

    # These all die on error:
    make_raid_device
    resync_device $NBD
    mount_raid_device

    echo "primary started."
    
    # Start Services - TODO: move this to a 'enbd-functions' file
    service p4d start
    die_if_not_ok $? "Error starting P4D!"
}

# Scenario 4 - Master fails
stop_primary() {
    remove_nbd_from_raid_device

    # Stop services - TODO: move this to a 'enbd-functions' file
    echo -n "0" > /proc/nbdinfo
    service p4d stop

    umount_raid_device
    stop_raid_device

    service enbd-client stop

    echo
    echo "WARNING:"
    echo "if this is a clean stop and you want to preserve data on your local"
    echo "device ($LOCAL_DEVICE), then you *MUST* 'rm $SERVER_FAILED_FILE'"
    echo "otherwise you'll end up syncing old data from the NBD!"
    echo
}
# Scenario 5 - Master revived
start_primary_after_failure() {
    service enbd-client start

#    echo -n "Rebuilding RAID-1 device & resyncing local device "

    # These all die on error:
    make_raid_device
    # assume that the local device is faulty:
    resync_device $LOCAL_DEVICE
    mount_raid_device

    echo "primary revived."

    # Start services - TODO: move this to a 'enbd-functions' file
    service p4d start
    die_if_not_ok $? "Error starting P4D!"
}

# Scenario 2 - Slave fails
primary_takeover_secondary() {
    remove_nbd_from_raid_device

    service enbd-client stop
    RETVAL=$?

    # RAID still writes to $LOCAL_DEVICE - leave it alone

    return $RETVAL
}

# Scenario 3 - Slave revived
primary_giveback_secondary() {
# TODO: heartbeat sometimes calls secondary stop on primary
# when primary has failed (even if it didn't have secondary
# resources...)
# so let's see if enbd-client is already running - if it is
# then we assume we're down.
# This doesn't work all the time :-/
    service enbd-client status
    ALIVE=$?
    if [ $ALIVE -eq 0 ]; then
    	echo "enbd-client is alive, assuming we're down."
    	echo "WARNING: cancelling give-back of secondary"
    	return
    fi

    service enbd-client start
    RETVAL=$?

    wait_until_nbd_is_up

    # TODO: sometimes this is done for us!
    # need to test if NBD is failed in the array...
    # can do this by looking at /proc/mdstat:
    #    md0 : active raid1 nda[1] hda8[0]
    #    md0 : active raid1 nda[1](F) hda8[0]

    #echo -n "Hot-adding RAID disk $NBD: "
    # this should force an 'intelligent' re-sync of $NDA
    #raidhotadd $RAID_DEVICE $NBD
#failure
#echo
#echo "hot-add $NBD disabled"
#    die_if_not_ok $?
printf "WARNING: Not hot-adding $NBD! \\n \
    this has been disabled -- usually the hot-add is done automatically \\n \
    need to automate this check: \\n \
	cat /proc/mdstat \\n \
    if you see this, it's already up: \\n \
	md0 : active raid1 nda[1] hda8[0] \\n \
    if you see this it's still down, and needs hot-adding: \\n \
	md0 : active raid1 nda[1](F) hda8[0] \\n \
    otherwise try doing 'raidhotadd $RAID_DEVICE $NBD' manually\\n"
}


# Assumption: /etc/raidtab is already configured to contain a RAID-1 device.
# Should include $LOCAL_DEVICE & $NBD.  If you want to use Fast Raid-1, don't
# forget to set "persistent-superblock 0"
make_raid_device() {
    # just in case it's already running:
    #eval umount_raid_device
    #eval stop_raid_device

    # TODO: took this out, as it was killing sshd ??!?
    # dunno why, it works as expected from cmdline...
    #fuser -muk $MOUNTPOINT
    umount $RAID_DEVICE
    raidstop $RAID_DEVICE

    wait_until_nbd_is_up

    echo -n "Starting RAID device $RAID_DEVICE: "

    modprobe md
    die_if_not_ok $? "Error loading 'md' kernel module!"
    modprobe fr1
    die_if_not_ok $? "Error loading 'fr1' kernel module!"

    mkraid --dangerous-no-resync --really-force $RAID_DEVICE
    # this sleeps for 5s
    die_if_not_ok $? "Error making RAID device $RAID_DEVICE!"

    success
    echo
}

remove_nbd_from_raid_device() {
    echo -n "Removing NBD from the RAID device: "
    echo -n "0" > /proc/nbdinfo
    raidsetfaulty $RAID_DEVICE $NBD && success || failure
    $RETVAL=$?
}

resync_device() {
    SYNC_DEVICE=$1
    echo -n "Resyncing RAID disk $SYNC_DEVICE: "

    raidsetfaulty $RAID_DEVICE $SYNC_DEVICE
    die_if_not_ok $? "Error setting $SYNC_DEVICE faulty!"

    echo
    echo "WARNING: forcing a complete re-sync"
    echo "Disabling FR-1 for the moment, as problems were occuring syncing local hd from nbd"
    echo
    raidhotremove $RAID_DEVICE $SYNC_DEVICE
    die_if_not_ok $? "Error hot-removing $SYNC_DEVICE!"

    # do we *really* need to do this?
    #raidstop $MD
    #sleep 1
    #raidstart $MD

    # this should force an 'intelligent' re-sync of $SYNC_DEVICE
    raidhotadd $RAID_DEVICE $SYNC_DEVICE
    die_if_not_ok $? "Error hot-adding $SYNC_DEVICE!"

    success
    echo
}

mount_raid_device() {
    echo -n "Mounting RAID device $RAID_DEVICE: "
    # TODO: see if $RAID_DEVICE is already mounted?
    # TODO: fsck $RAID_DEVICE
    mount -t $FS_TYPE $RAID_DEVICE $MOUNTPOINT $MOUNT_OPTS
    die_if_not_ok $? "Error mounting $RAID_DEVICE!"
    success
    echo
}

umount_raid_device() {
    echo -n "Unmounting RAID device $RAID_DEVICE: "
    # kick everybody off:
#    fuser -muk $MOUNTPOINT
    # removed: kills all processes?  why?  works manually...

    umount $RAID_DEVICE
    die_if_not_ok $? "Error umounting $RAID_DEVICE!"

    success
    echo
}

stop_raid_device() {
    echo -n "Stopping RAID device $RAID_DEVICE: "
    raidstop $RAID_DEVICE
    die_if_not_ok $? "Error stopping RAID device $RAID_DEVICE!"
    success
    echo
}

wait_until_nbd_is_up() {
    echo "Waiting until NBD is available..."

    # Explanation: we can test to see if $NBD is up by looking in /proc/nbdinfo
    # for lines like:
    #    "Device a:       Open"
    #    "[a] Cpids:      0       (0)     (0)"
    # (NOTE: this may not work all the time, unless we always 'rmmod enbd'
    #  when shutting enbd-client down)
    while [ 1 ]; do
	# TODO: this won't work if we have 2 diff clients running,
	#       would be better to check for device letter here...
	service enbd-client status
	CLIENT_ALIVE=$?
	die_if_not_ok $CLIENT_ALIVE \
	    "Nothing to wait for: enbd-client is not running!"

	ENBD_PIDS=`grep 'Cpids' /proc/nbdinfo | cut -f2`
	if [ "$ENBD_PIDS" -gt 0 ]; then
	    return 0
	fi
	sleep 1
    done

    echo -n "NBD available: "
    success
    echo
}

die_if_not_ok() {
    RETVAL=$1
    ERROR_MSG=$2
    if [ $RETVAL -ne 0 ]; then
	failure
	echo
	printf "$ERROR_MSG"
	echo
	exit $RETVAL
    fi
}

