#!/bin/bash
#
#  Periodically monitor for jobs which has finished or failed but not
#  reported an exitcode
#
#set -x 
id=`id -u`

#debug=:
debug () {
    echo -n `date` 1>&2
    echo -n ' ' 1>&2
    echo $@ 1>&2
}

debug "starting"
debug "options = $@"

# ARC1 passes first the config file.
if [ "$1" = "--config" ]; then shift; ARC_CONFIG=$1; shift; fi
if [ -z "$1" ] ; then echo "Controldir argument missing" 1>&2 ; exit 1 ; fi

joboption_lrms="boinc"
lrms_options="boinc_app_id boinc_db_host boinc_db_port boinc_db_user boinc_db_pass boinc_db_name boinc_project_dir"

# define paths and config parser
basedir=`dirname $0`
basedir=`cd $basedir > /dev/null && pwd` || exit $?
. "${basedir}/lrms_common.sh"

# include common scan functions
. "${pkgdatadir}/scan_common.sh" || exit $?

# run common init 
#  * parse config
#  * load LRMS-specific env
#  * set common variables
common_init

# Define gm-kick location
GMKICK=${pkglibexecdir}/gm-kick

for control_dir in "$@" ; do

    if [ ! -d "${control_dir}" ]; then 
        echo "No control dir $control_dir" 1>&2
        continue
    fi

    # Bash specific, but this script will be rewritten in python soon...
    declare -A finished_jobs
    wuappidclause=""
    if [ ! -z "$CONFIG_boinc_app_id" ]; then
        wuappidclause="and wu.appid=$CONFIG_boinc_app_id"
    fi
    finished=$(mysql -h $CONFIG_boinc_db_host -P $CONFIG_boinc_db_port -u $CONFIG_boinc_db_user --password=$CONFIG_boinc_db_pass $CONFIG_boinc_db_name -N -e "select wu.name from workunit wu where wu.assimilate_state=2 $wuappidclause")

    for job in `echo $finished`; do
        finished_jobs[$job]=1
    done

    # iterate over all jobs known in the control directory
    find "${control_dir}/processing" -name '*.status' \
    | xargs egrep -l "INLRMS|CANCELING" \
    | sed -e 's/.*\///' -e 's/\.status$//' \
    | while read job; do
        #debug "scanning job = $job"
        unset joboption_jobid
        unset joboption_directory

        lrmsfile=$(control_path "${control_dir}" "${job}" "lrms_done")
        gramifile=$(control_path "${control_dir}" "${job}" "grami")
        localfile=$(control_path "${control_dir}" "${job}" "local")
        errorsfile=$(control_path "${control_dir}" "${job}" "errors")

        # this job was already completed, nothing remains to be done
        [ -f "$lrmsfile" ] && continue

        # a grami file exists for all jobs that GM thinks are running.
        # proceed to next job if this file is missing.
        if [ ! -f "$gramifile" ]; then
            continue
        fi

        # extract process IDs of the grami file
        [ ! -f "$gramifile" ] && continue
        . "$gramifile"

        # process IDs could not be learned, proceeding to next
        [ -z "$joboption_jobid" ] && continue

        #debug "local jobid = $joboption_jobid"

        # checking if process is still running
        if [[ ! ${finished_jobs[$joboption_jobid]} ]]; then
            #debug "$joboption_jobid is still running, Continuing to next"
            continue
        else
            debug "$joboption_jobid is finished"
        fi
        uid=$(get_owner_uid "$localfile")
        sessiondir=${joboption_directory}
        debug "local user id = $uid"
        diagfile=${joboption_directory}.diag
        commentfile=${joboption_directory}.comment

        debug "checking $diagfile"
        job_read_diag

        comment=""
        error_mask=$(mysql -h $CONFIG_boinc_db_host -P $CONFIG_boinc_db_port -u $CONFIG_boinc_db_user --password=$CONFIG_boinc_db_pass $CONFIG_boinc_db_name -N -e "select error_mask from workunit wu where wu.name='$joboption_jobid'")
        debug "got error_mask=$error_mask"

        if [ "x$error_mask" != "x0" ] ; then
            comment="BOINC server reported error mask $error_mask"
            sep=":"
            if [ $(( error_mask & 0x01 )) -ne 0 ] ; then comment="$comment$sep Couldn't send result"; sep=","; fi
            if [ $(( error_mask & 0x02 )) -ne 0 ] ; then comment="$comment$sep Too many error results"; sep=","; fi
            if [ $(( error_mask & 0x04 )) -ne 0 ] ; then comment="$comment$sep Too many success results"; sep=","; fi
            if [ $(( error_mask & 0x08 )) -ne 0 ] ; then comment="$comment$sep Too many total results"; sep=","; fi
            if [ $(( error_mask & 0x10 )) -ne 0 ] ; then comment="$comment$sep Workunit cancelled"; sep=","; fi
            if [ $(( error_mask & 0x20 )) -ne 0 ] ; then comment="$comment$sep No canonical result"; fi
            exitcode=-1

            do_as_uid "$uid" "touch '$commentfile'"
        else
            exit_status=""
            sent_time=""
            recv_time=""
            elapsed_time=""
            cpu_time=""
            wss=""

            IFS=$'\t' read -r exit_status sent_time recv_time elapsed_time cpu_time wss <<END
`mysql -h $CONFIG_boinc_db_host -P $CONFIG_boinc_db_port -u $CONFIG_boinc_db_user --password=$CONFIG_boinc_db_pass $CONFIG_boinc_db_name -N -r -e "select r.exit_status, r.sent_time, r.received_time, cast(r.elapsed_time as unsigned), cast(r.cpu_time as unsigned), cast(r.peak_working_set_size as unsigned) from workunit wu join result r on r.id=wu.canonical_resultid where wu.name='$joboption_jobid' and wu.canonical_resultid > 0 $wuappidclause"`
END

            LRMSExitcode="$exit_status"
            comment="Job completed successfully"

            if [ "x$sent_time" != "x0" -a "x$recv_time" != "x0" ] ; then
                seconds_to_mds_date $sent_time
                LRMSStartTime=$return_mds_date

                seconds_to_mds_date $recv_time
                LRMSEndTime=$return_mds_date
            fi

            WallTime=${WallTime%%.*}
            if [ "x$elapsed_time" != "x0" ] ; then
                if [ -z "$WallTime" ] ; then
                    WallTime=$elapsed_time
                elif [ $elapsed_time -gt $WallTime ] ; then
                    WallTime=$elapsed_time
                fi
            fi

            if [ "x$cpu_time" != "x0" ] ; then
                KernelTime=0
                UserTime=$cpu_time
            fi

            if [ "x$wss" != "x0" ] ; then
                TotalMemory=$(( wss / 1024 ))
            fi

            #debug "name=$joboption_jobname"
            #debug "got LRMSExitcode=$LRMSExitcode LRMSStartTime=$LRMSStartTime LRMSEndTime=$LRMSEndTime WallTime=${WallTime}s UserTime=${UserTime}s TotalMemory=${TotalMemory}kB"

            # extract stdout
            mysql -h $CONFIG_boinc_db_host -P $CONFIG_boinc_db_port -u $CONFIG_boinc_db_user --password=$CONFIG_boinc_db_pass $CONFIG_boinc_db_name -N -r -e "select stderr_out from workunit wu join result r on r.id=wu.canonical_resultid where wu.name='$joboption_jobid' and wu.canonical_resultid > 0 $wuappidclause" \
            | /usr/bin/perl -0777 -ne 'print $1 if /<stderr_txt>(.*?)<\/stderr_txt>/s' | sed 's/\r$//' \
            | do_as_uid "$uid" "cat >'$commentfile'"

            # try to fix processor count
            processors_diag=$(do_as_uid "$uid" "cat '$diagfile'" | sed -n 's/^Processors=\([0-9]*\).*/\1/p' | head -n1)
            if [ -z "$processors_diag" ] ; then
                processors_diag=${joboption_count:-1}
            fi

            # real core count used is minimum from vboxwrapper and athena
            processors_vbox=$(do_as_uid "$uid" "cat '$commentfile'" | sed -n 's/^.* Setting CPU Count for VM\. (\([0-9]*\))$/\1/p' | head -n1)
            processors_athena=$(do_as_uid "$uid" "cat '$commentfile'" | sed -n 's/^.* ATHENA_PROC_NUMBER=\([0-9]*\)$/\1/p' | head -n1)
            if [ -z "$processors_athena" ] ; then
                processors_athena=$(do_as_uid "$uid" "cat '$commentfile'" | sed -n 's/^.* Threads: \([0-9]*\)$/\1/p' | head -n1)
            fi
            debug "got core count from vbox=[$processors_vbox] athena=[$processors_athena]"
            if [ -n "$processors_vbox" -a -n "$processors_athena" ] ; then
                if [ $processors_vbox -lt $processors_athena ] ; then
                    processors_log=$processors_vbox
                else
                    processors_log=$processors_athena
                fi
            else
                processors_log=$processors_vbox$processors_athena
            fi

            if [ -n "$processors_log" -a "x$processors_diag" != "x$processors_log" ] ; then
                debug "Fixing Processors in diagfile [$processors_diag] to detected from log [$processors_log]"
                Processors=$processors_log
            else
                Processors=$processors_diag
            fi
            do_as_uid "$uid" "sed -i '/^Processors=/d' '$diagfile'"

            #debug "got Processors=$Processors Efficiency=$(( ${UserTime} * 100 / ( ${WallTime} * ${Processors} ) ))%"

            # no exitcode in diagfile, assume successful job termination
            if [ -z "$exitcode" ] ; then
                exitcode=0
            fi
        fi

        if [ -z "$joboption_arg_code" ] ; then joboption_arg_code='0' ; fi
        if [ -z "$exitcode" ]; then
            echo "Job $job with PID $joboption_jobid died unexpectedly" 1>&2
            comment="Job died unexpectedly"
            exitcode=-1
        elif [ "$exitcode" -ne "$joboption_arg_code" ]; then
            comment="Job finished with wrong exit code - $exitcode != $joboption_arg_code"
        fi
        debug "got exitcode=$exitcode comment=$comment"

        job_write_diag

        save_commentfile "$uid" "$commentfile" "$errorsfile"
        echo "$exitcode $comment" > "$lrmsfile"
        "${GMKICK}" -j "${job}" "${control_dir}"
    done

done

debug "done, going to sleep"

sleep 120
exit 0
