#!/bin/bash
#================================================================================
# File:		vdb_action.sh
# Type:	 	bash-shell script
# Date:	 	26-July 2019
# Author:	Delphix Field Services
# Ownership:	This script is owned and maintained by the user, not by Delphix
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Copyright (c) 2019 by Delphix. All rights reserved.
# 
# Description:
#
#	This is an "action script" for an Oracle Clusterware customized resource
#	to perform actions initiated by Oracle Clusterware (CRS) for a "Generic
#	Resource".
#
#	The recommended syntax for creating the customized generic resource is:
#
#		crsctl add resource <resource-name> \
#			-type generic_application \
#			-attr "SERVER_POOLS=Generic,\
#				START_PROGRAM='<full-path-to-this-script> start <db-name>',\
#				STOP_PROGRAM='<full-path-to-this-script> stop <db-name>',\
#				CHECK_PROGRAMS='<full-path-to-this-script> check <db-name>',\
#				CLEAN_PROGRAM='<full-path-to-this-script> clean <db-name>',\
#				CARDINALITY=%CRS_SERVER_POOL_SIZE%,\
#				AUTO_START=restore,\
#				SCRIPT_TIMEOUT=75
#
#	where...
#
#		<resource-name>			CRS resource name (no spaces) 
#		<full-path-to-this-script>	full path to this shell script on
#						all RAC cluster nodes
#		<db-name>			value of DB_UNIQUE_NAME initialization parameter
#
# Note:
#
#	CRSCTL commands can be executed from any one of the nodes in the RAC cluster
#	while connected to the shell as the Oracle clusterware owner OS account
#	(i.e. "grid", "oracle").
#
# Standard Oracle documentation:
#
#	Entitled "Making Applications Highly Available Using Oracle Clusterware"
#	for version 12.2 can be found at...
#
#		https://docs.oracle.com/en/database/oracle/oracle-database/12.2/cwadd/making-applications-highly-available-using-oracle-clusterware.html#GUID-13400D83-3FF4-482E-B7A7-41C864A491DC
#
# Calling syntax:
#
#       vdb_action.sh action-type db-name
#
# Calling parameters:
#
#	action-type	one of the following:  start, stop, check, clean
#	db-name		value of DB_UNIQUE_NAME initialization parameter
#
# Modifications:
#	TGorman	26jul19	written
#	TGorman	19aug19	added db-name parameter
#================================================================================
#
#--------------------------------------------------------------------------------
# Verify that there are two command-line parameters...
#--------------------------------------------------------------------------------
if (( $# != 2 ))
then
	echo "`date` [ERROR] Usage: \"$0 action-type db-name\"; aborting..."
	exit 1
fi
#
#--------------------------------------------------------------------------------
# Please customize the following variables for local preferences...
#--------------------------------------------------------------------------------
_dbName=$2
_dlpxEngine="dv-wodc.fda.gov"
_baseMntPath="/delphix/provision"
_logDir="/tmp"
_dlpxUser="mgmt_`hostname -s`"
#
#--------------------------------------------------------------------------------
# Please do not change the following script variables unless prepared to make
# changes elsewhere to the script...
#--------------------------------------------------------------------------------
_hostName="`hostname -s`"
_fileLabel="delphix_monitor"
_monitorVdbDir=${_baseMntPath}/${_dbName}/datafile
_monitorVdbFile=${_monitorVdbDir}/${_fileLabel}_${_dbName}_${_hostName}.txt
_logFile=${_logDir}/${_fileLabel}_${1}_${_dbName}_${_hostName}_$$.txt
typeset -i _loopSleep=3
typeset -i _loopLimit=100
#
#--------------------------------------------------------------------------------
# Clean out previous log files...
#--------------------------------------------------------------------------------
rm -f /tmp/${_fileLabel}_${1}_${_dbName}_${_hostName}_*.txt
#
#--------------------------------------------------------------------------------
# perform START, STOP/CLEAN, and CHECK operations...
#--------------------------------------------------------------------------------
case "$1" in
	#------------------------------------------------------------------------
	# START operation will first connect to the Delphix virtualization engine
	# and attempt to START the VDB...
	#------------------------------------------------------------------------
	start)		ssh ${_dlpxUser}@${_dlpxEngine} << __EOF__ 2>&1 >> ${_logFile}
/source select "${_dbName}" start; commit; exit
__EOF__
			#
			#--------------------------------------------------------
			# if the output from the Delphix CLI command includes the
			# phrase "Error:", then it is only possibly an actual
			# error...
			#--------------------------------------------------------
			if grep "Error:" ${_logFile} > /dev/null 2>&1
			then
				#
				#------------------------------------------------
				# if the output from the Delphix CLI command also
				# includes the verbiage "only one job can be active",
				# then it is not really an error, just a concurrency
				# problem with multiple nodes sending a START at the
				# same time...
				#------------------------------------------------
				if grep -i "only one job can be active for an object" ${_logFile} > /dev/null 2>&1
				then
					typeset -i _loopCnt=0
					while (( ${_loopCnt} < ${_loopLimit} ))
					do
						if [ -d ${_monitorVdbDir} ]
						then
							break
						else
							sleep ${_loopSleep}
						fi
						typeset -i _loopCnt=${_loopCnt}+1
					done
					if (( ${_loopCnt} >= ${_loopLimit} ))
					then
						echo "`date` [ERROR] \"$0 $1 $2\" on \"${_hostName}\" - mount failure" | tee -a ${_logFile}
						exit 1
					fi
				else
					echo "`date` [ERROR] \"$0 $1 $2\" on \"${_hostName}\" - CLI failure" | tee -a ${_logFile}
					exit 1
				fi
				echo "`date` [ERROR] \"$0 $1 $2\" on \"${_hostName}\" - duplicate, success" | tee -a ${_logFile}
			else
				echo "`date` [ERROR] \"$0 $1 $2\" on \"${_hostName}\" - success" | tee -a ${_logFile}
			fi
			#
			#----------------------------------------------------------
			# If the START operation via the Delphix CLI succeeded, then
			# put a timestamp into the "monitor file" within the NFS
			# mountpoint in which the VDB resides...
			#----------------------------------------------------------
			if date > ${_monitorVdbFile} 2>> ${_logFile}
			then
				echo "`date` [INFO] \"$0 $1 $2\" success on \"${_hostName}\"" | tee -a ${_logFile}
				exit 0
			else
				echo "`date` [ERROR] \"$0 $1 $2\" on \"${_hostName}\" - date failure" | tee -a ${_logFile}
				exit 1
			fi
			;;
	#------------------------------------------------------------------------
	# STOP and CLEAN are the same actions...
	#
	# If the "monitor file" within the NFS mountpoint in which the VDB resides
	# is accessible, then remove it and then contact the Delhpix virtualization
	# engine in order to STOP the VDB...
	#------------------------------------------------------------------------
	stop|clean)	if rm -f ${_monitorVdbFile} 2>&1 >> ${_logFile}
			then
				#
				#------------------------------------------------
				# if the "rm -f" command succeeded, then the NFS
				# mount-point is still mounted and the VDB instance
				# needs to be stopped...
				#------------------------------------------------
				ssh ${_dlpxUser}@${_dlpxEngine} << __EOF__ 2>&1 >> ${_logFile}
/source select "${_dbName}"; stop; commit; exit
__EOF__
				if grep "Error:" ${_logFile} > /dev/null 2>&1
				then
					if grep -i "only one job can be active for an object" ${_logFile} > /dev/null 2>&1
					then
						typeset -i _loopCnt=0
						while (( ${_loopCnt} < ${_loopLimit} ))
						do
							if [ -d ${_monitorVdbDir} ]
							then
								break
							else
								sleep ${_loopSleep}
							fi
							typeset -i _loopCnt=${_loopCnt}+1
						done
						if (( ${_loopCnt} >= ${_loopLimit} ))
						then
							echo "`date` [ERROR] \"$0 $1 $2\" on \"${_hostName}\" - mount-point failure" | tee -a ${_logFile}
							exit 1
						fi
					else
						echo "`date` [ERROR] \"$0 $1 $2\" on \"${_hostName}\" - CLI failure" | tee -a ${_logFile}
						exit 1
					fi
					echo "`date` [ERROR] \"$0 $1 $2\" on \"${_hostName}\" - duplicate, success" | tee -a ${_logFile}
				else
					echo "`date` [ERROR] \"$0 $1 $2\" on \"${_hostName}\" - success" | tee -a ${_logFile}
				fi
			else
				echo "`date` [ERROR] \"$0 $1 $2\" on \"${_hostName}\" - rm failure" | tee -a ${_logFile}
				echo "\"rm -f ${_monitorVdbFile}\" failed" >> ${_logFile}
				exit 1
			fi
			echo "`date` [INFO] \"$0 $1 $2\" success on \"${_hostName}\"" | tee -a ${_logFile}
			exit 0
			;;
	#------------------------------------------------------------------------
	# CHECK operation involves simply checking for the accessibility of the
	# "monitor" file residing within the NFS mount-point of the VDB...
	#------------------------------------------------------------------------
	check)		if [ -e ${_monitorVdbFile} ]
			then
				echo "`date` [INFO] \"$0 $1 $2\" success on \"${_hostName}\"" | tee -a ${_logFile}
				exit 0
			else
				echo "`date` [ERROR] \"$0 $1 $2\" failure on \"${_hostName}\"" | tee -a ${_logFile}
				exit 1
			fi
			;;
	#------------------------------------------------------------------------
	# invalid command-line parameter passed to the script...
	#------------------------------------------------------------------------
	*)		echo "`date` [ERROR] \"$0 $1 $2\" on \"${_hostName}\" - invalid action requested" | tee -a ${_logFile}
			exit 1
			;;
esac