# DCcrs.ctl:400:Collects Cluster Information
# $Id: DCcrs.ctl,v 1.13 2015/08/21 15:34:34 RDA Exp $
# ARCS: $Header: /home/cvs/cvs/RDA_8/src/scripting/lib/collect/DB/DCcrs.ctl,v 1.13 2015/08/21 15:34:34 RDA Exp $
#
# Change History
# 20150821  MSC  Improve time consistency.

=head1 NAME

DB:DCcrs - Collects Cluster Information

=head1 DESCRIPTION

This module collects data that is related to the operation of the Oracle
Cluster and its configuration.

The following reports can be generated and are regrouped under C<Cluster
Information> in C<Cluster>:

=cut

use Mrc

echo tput('bold'),'Processing DB.CRS module ...',tput('off')

# Initialization
var $AGE         = ${R_FILE_AGE/T:15}
var $CRS_HOME    = ${D_CRS_HOME}
var $CRS_TAIL    = ${N_TAIL:100}
var $LOG_TAIL    = ${N_LOG_TAIL:2000}
var $ORACLE_HOME = ${SET.RDA.BEGIN.D_ORACLE_HOME:''}

var $CORE = '(^core\.\d+$|core$)'
var $OSN  = getOsName()
var $PAT1 = concat('itm',$AGE)
var $PAT2 = concat('dirm',$AGE)
var $PAT3 = concat('drm',$AGE)
var $R112 = false
var $R12  = false
var $R9   = false

var $MOD = cond(isUnix(),'fx','fr')

var $TOC = '%TOC%'
var $TOP = '[[#Top][Back to top]]'
toc '^1:Cluster'
toc '1+:Cluster Information'

var $WARN = '   * Links point to files that have been collected in their \
                  original format. Opening them directly in your browser can \
                  present risks. To prevent them, access the file outside the \
                  browser or use the link to save them and use an adequate \
                  viewer.'

# Load the common macros
run DB:DBinfo()
run RDA:INVinfo()
run RDA:library()

# Set the database context
if ${I_DBC}
 call setSqlTarget(last)

=pod

Reports the details of the operating system-specific aspects (platform-specific
code)

=cut

if !?$CRS_HOME
 run DB:CRSinit(\$CRS_HOME)

var @CFG = ()
var @LOG = ()
var @INITD = ()
if match($OSN,'solaris')
 run OS:OSsunos('CRS',\@INITD,\@CFG,\@LOG)
elsif match($OSN,'hpux')
 run OS:OShpux('CRS',\@INITD,\@CFG,\@LOG)
elsif match($OSN,'aix')
 run OS:OSaix('CRS',\@INITD,\@CFG,\@LOG)
elsif match($OSN,'dec_osf')
 run OS:OSosf('CRS',\@INITD,\@CFG,\@LOG)
elsif match($OSN,'linux')
 run OS:OSlinux('CRS',\@INITD,\@CFG,\@LOG)
elsif or(isWindows(),isCygwin())
 run OS:OSwin32('CRS')
elsif !isVms()
 echo 'Platform ',$OSN,' not supported'

=head2 racOnOff - RAC on/off Linked

Verifies that the RAC option is currently linked into the F<oracle> executable.
This check is performed for supported UNIX platforms only.

=cut

if match($OSN,'aix|dec_osf|hpux|linux|solaris')
{debug ' Inside CRS module, checking ORACLE binaries to see if RAC is linked'
 report racOnOff
 write '---+ Verify RAC Linked'
 var $cmd = concat('nm -r ',cond(match($OSN,'aix'),'-X 64 ',''),\
                    catCommand($ORACLE_HOME,'rdbms','lib','libknlopt.a'))
 write '---## Checking ORACLE binaries to see if RAC is linked'
 write '---### Using command: ',encode($cmd),' | grep kcsm.o'
 if grepCommand(concat($cmd,' 2>/dev/null'),'kcsm\.o','f')
  write 'RAC option IS installed and linked into the Oracle binary.'
 else
  write 'RAC NOT correctly linked or installed into the Oracle binary'
 toc '2:[[',getFile(),'][rda_report][RAC on/off Linked]]'
}

=for stopwords Init init

=head2 init - Init Files

For UNIX, the report collects the F</etc/inittab> and the init files. Additional
platform-specific files can be present.

=cut

if isUnix()
{debug ' Inside CRS module, gathering init files'
 report init
 prefix
 {write '---+!! Init Files'
  write $TOC
 }
 loop $fil (@INITD)
 {if ?testFile('f',$fil)
  {var $nam = basename($fil)
   write '---+ Display of ',encode($nam),' File'
   write '---## Information Taken from ',encode($fil)
   if !writeFile($fil)
   {write '**',encode($nam),' not readable.**%BR%\
          May be file permission problems.%BR%\
          Permissions are:%BR%'
    call statFile('b',$fil)
    write 'User: ',id(),'%BR%'
   }
   write $TOP
  }
 }
 if isCreated(true)
  toc '2:[[',getFile(),'][rda_report][Init Files]]'

=head2 core - Core Dumps

When a debugger is found for UNIX, RDA tries to extract the stack trace from
the core dumps present in the cluster directories.

=cut

 if $CRS_HOME
 {debug ' Inside CRS module, detecting core dumps'

  report core
  var $log = catDir($CRS_HOME,'log',${RDA.T_NODE})
  var @dmp = (grepDir(catDir($CRS_HOME,'crs','init'),$CORE,$PAT3),\
              grepDir(catDir($CRS_HOME,'css','init'),$CORE,$PAT3),\
              grepDir(catDir($CRS_HOME,'evm','init'),$CORE,$PAT3),\
              grepDir(catDir($log,'crsd'),$CORE,$PAT3),\
              grepDir(catDir($log,'cssd'),$CORE,$PAT3),\
              grepDir(catDir($log,'evmd'),$CORE,$PAT3))
  var $flg = true
  if @dmp
  {# Include the core analyzer
   run OS:COREinfo()
   var $flg = false

   # When a debugger is found, analyze the core dumps
   if can_analyze_core()
   {var $dbg = last

    write '---+!! Core Dump Stack Trace Extraction'
    write '---## Using: ',encode($dbg)
    write $TOC
    call analyze_core(@dmp)
   }
  }
  if match(getOsName(),'solaris')
  {if $flg
    run OS:COREinfo()
   call run_coreadm(('crsd|evmd|evmlogger|init\.cssd|ocssd|ons|oprocd|\
                      racgmain|racgimon'))
  }
  if isCreated(true)
   toc '2:[[',getFile(),'][rda_report][Core Dumps]]'
 }
}

=head2 crs_status - CRS Status

Sets up and populates CRS_status if Cluster Ready Services is installed.

=cut

debug ' Inside CRS module, checking Cluster Ready Services status'
report crs_status
write '---+!! Cluster Ready Services Status'
write $TOC

prefix
 write 'Cluster Ready Services detected.'
if $CRS_HOME
{if ?testFile('f',catFile($CRS_HOME,'bin',${AS.EXE:'olsnodes'}))
 {write '---+ Nodes Found'
  write '---## Using: olsnodes -n -i'
  if !isVms()
  {call writeCommand(concat(lastCommand(),' -n -i 2>&1'))
   write $TOP
  }
  elsif ?testFile('f',catFile($ORACLE_HOME,'bin','olsnodes.exe'))
  {call writeCommand('olsnodes -n -i')
   write $TOP
  }
  else
   write '---## Only available for CRS code trees'
 }
 if ?testFile('f',catFile($CRS_HOME,'bin',${AS.EXE:'crsctl'}))
  var $CRSCTL = cond(isVms(),undef,lastCommand())
}

if !?$CRSCTL
{if ?testFile('f',catFile($ORACLE_HOME,'bin',${AS.EXE:'crsctl'}))
  var $CRSCTL = cond(isVms(),'crsctl',lastCommand())
}
if !hasOutput(true)
{write
 write '**Cluster Ready Services NOT detected.**%BR%'
}

if $CRSCTL
{prefix
 {write '---+ Cluster Synchronization Services Daemon Status'
  write '---## Using: crsctl check css'
 }
 call writeCommand(concat($CRSCTL,' check css'))
 if hasOutput(true)
  write $TOP

 # Perform on 11.2 or later crsctl collections
 if grepCommand(concat($CRSCTL,' query crs softwareversion'),\
                '(11\.2|12\.[12])(\.\d+){3}','f1')
 {if compare('VALID',first,'12')
   var $R12 = true
  else
   var $R112 = true

  # Perform 12.x crsctl collections
  if $R12
  {prefix
   {write '---+ Cluster Hub Size'
    write '---## Using: crsctl get cluster hubsize'
   }
   call writeCommand(concat($CRSCTL,' get cluster hubsize'))
   if hasOutput(true)
    write $TOP

   prefix
   {write '---+ Cluster Mode Configuration Information'
    write '---## Using: crsctl get cluster mode config'
   }
   call writeCommand(concat($CRSCTL,' get cluster mode config'))
   if hasOutput(true)
    write $TOP

   prefix
   {write '---+ Cluster Mode Status Information'
    write '---## Using: crsctl get cluster mode status'
   }
   call writeCommand(concat($CRSCTL,' get cluster mode status'))
   if hasOutput(true)
    write $TOP

   prefix
   {write '---+ Cluster Leaf Miscount Information'
    write '---## Using: crsctl get css leafmisscount'
   }
   call writeCommand(concat($CRSCTL,' get css leafmisscount'))
   if hasOutput(true)
    write $TOP

   prefix
   {write '---+ Cluster Node Role Configuration Information'
    write '---## Using: crsctl get node role config -all'
   }
   call writeCommand(concat($CRSCTL,' get node role config -all'))
   if hasOutput(true)
    write $TOP

   prefix
   {write '---+ Cluster Node Role Status Information'
    write '---## Using: crsctl get node role status -all'
   }
   call writeCommand(concat($CRSCTL,' get node role status -all'))
   if hasOutput(true)
    write $TOP

   prefix
   {write '---+ Cluster Release Patch Information'
    write '---## Using: crsctl query crs releasepatch'
   }
   call writeCommand(concat($CRSCTL,' query crs releasepatch'))
   if hasOutput(true)
    write $TOP

   prefix
   {write '---+ Cluster Software Patch Information'
    write '---## Using: crsctl query crs softwarepatch'
   }
   call writeCommand(concat($CRSCTL,' query crs softwarepatch'))
   if hasOutput(true)
    write $TOP

   prefix
   {write '---+ Cluster Socket Information'
    write '---## Using: crsctl query socket udp'
   }
   call writeCommand(concat($CRSCTL,' query socket udp'))
   if hasOutput(true)
    write $TOP
  }

  # Perform common collections
  prefix
  {write '---+ List of Users with Oracle Clusterware Administrative Privileges'
   write '---## Using: crsctl query crs administrator'
  }
  call writeCommand(concat($CRSCTL,' query crs administrator'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ Intelligent Platform Management Interface (IPMI) Configuration \
               Information'
   write '---## Using: crsctl query css ipmiconfig'
  }
  call writeCommand(concat($CRSCTL,' query css ipmiconfig'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ Intelligent Platform Management Interface (IPMI) Driver \
               Information'
   write '---## Using: crsctl query css ipmidevice'
  }
  call writeCommand(concat($CRSCTL,' query css ipmidevice'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ Domain Name System (DNS) Server Information'
   write '---## Using: crsctl query dns -servers'
  }
  call writeCommand(concat($CRSCTL,' query dns -servers'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ Address of Local Intelligent Platform Management Interface \
              (IPMI) Device'
   write '---## Using: crsctl get css ipmiaddr'
  }
  call writeCommand(concat($CRSCTL,' get css ipmiaddr'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ Cluster Priority'
   write '---## Using: crsctl get css priority'
  }
  call writeCommand(concat($CRSCTL,' get css priority'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ Status of Cluster Time Synchronization Services'
   write '---## Using: crsctl check ctss'
  }
  call writeCommand(concat($CRSCTL,' check ctss'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ ora.crsd Resource Status and Configuration'
   write '---## Using: crsctl status resource ora.crsd -init'
  }
  call writeCommand(concat($CRSCTL,' status resource ora.crsd -init'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ ora.crsd Resource Log Level'
   write '---## Using: crsctl get log res ora.crsd -init'
  }
  call writeCommand(concat($CRSCTL,' get log res ora.crsd -init'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ High Availability Services and Clusterware Stack Status'
   write '---## Using: crsctl check crs'
  }
  call writeCommand(concat($CRSCTL,' check crs'))
  if hasOutput(true)
   write $TOP
 }
 else
 {# Perform pre 11.2 crsctl collections
  prefix
  {write '---+ Cluster Ready Services Status'
   write '---## Using: crsctl check crs status'
  }
  call writeCommand(concat($CRSCTL,' check crs status'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ Cluster Ready Services Viability'
   write '---## Using: crsctl check crsd status'
  }
  call writeCommand(concat($CRSCTL,' check crsd status'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ Cluster Synchronization Services Viability'
   write '---## Using: crsctl check cssd status'
  }
  call writeCommand(concat($CRSCTL,' check cssd status'))
  if hasOutput(true)
   write $TOP

  prefix
  {write '---+ Event Manager Viability'
   write '---## Using: crsctl check evmd status'
  }
  call writeCommand(concat($CRSCTL,' check evmd status'))
  if hasOutput(true)
   write $TOP
 }
 prefix
 {write '---+ Location of the Vote Disks'
  write '---## Using: crsctl query css votedisk'
 }
 call writeCommand(concat($CRSCTL,' query css votedisk'))
 if hasOutput(true)
  write $TOP

 prefix
 {write '---+ CSS Daemon Priority'
  write '---## Using: crsctl get css priority'
 }
 call writeCommand(concat($CRSCTL,' get css priority'))
 if hasOutput(true)
  write $TOP

 prefix
 {write '---+ CSS Diagwait Status'
  write '---## Using: crsctl get css diagwait'
 }
 call writeCommand(concat($CRSCTL,' get css diagwait'))
 if hasOutput(true)
  write $TOP

 prefix
 {write '---+ Cluster Disk Latencies'
  write '---## Using: crsctl get css disktimeout'
 }
 call writeCommand(concat($CRSCTL,' get css disktimeout'))
 if hasOutput(true)
  write $TOP

 prefix
 {write '---+ Cluster Network Latencies'
  write '---## Using: crsctl get css misscount'
 }
 call writeCommand(concat($CRSCTL,' get css misscount'))
 if hasOutput(true)
  write $TOP

 prefix
 {write '---+ Cluster Reboot Time'
  write '---## Using: crsctl get css reboottime'
 }
 call writeCommand(concat($CRSCTL,' get css reboottime'))
 if hasOutput(true)
  write $TOP

 prefix
 {write '---+ Cluster Ready Services Software Version'
  write '---## Using: crsctl query crs softwareversion'
 }
 call writeCommand(concat($CRSCTL,' query crs softwareversion'))
 if hasOutput(true)
  write $TOP

 prefix
 {write '---+ Cluster Ready Services Active Version'
  write '---## Using: crsctl query crs activeversion'
 }
 call writeCommand(concat($CRSCTL,' query crs activeversion'))
 if hasOutput(true)
  write $TOP
}

# Get ocr.loc
if or(isWindows(),isCygwin())
{write '---+ OCR Configuration Registry Contents'
 if !writeRegistry('HKLM\SOFTWARE\Oracle\Ocr')
  write '**There are no OCR registry entries.**%BR%'
}
else
{write '---+ OCR Configuration File Contents'
 if ?testFile('f','/var/opt/oracle/ocr.loc')
  call writeFile('/var/opt/oracle/ocr.loc')
 elsif ?testFile('f','/etc/oracle/ocr.loc')
  call writeFile('/etc/oracle/ocr.loc')
 elsif isVms()
 {var $hom = getEnv('SYS$LOGIN')
  var ($nod) = command(concat('write sys$output f$getsyi("NODENAME")'))
  loop $dir (catDir($hom,$nod,'oracle'))
  {if ?testFile('r',catFile($dir,'ocr.loc'))
   {call writeFile(lastFile())
    break
   }
  }
 }
 else
  write '**ocr.loc file not found.**%BR%'
}
write $TOP
toc '2:[[',getFile(),'][rda_report][CRS Status]]'

=head2 cluvfy - Cluster Verify Information

Collects Cluster Verify information.

=cut

if isVms()
 var $pgm = undef
elsif ?testFile($MOD,catFile($CRS_HOME,'bin',${AS.BATCH:'cluvfy'}))
 var $pgm = lastCommand()
elsif ?findCommand('cluvfy')
 var $pgm = last
else
 var $pgm = undef
if ?$pgm
{debug ' Inside CRS module, getting Cluster Verify information'
 report cluvfy
 write '---+!! Cluster Verify Information'
 write $TOC
 var $cmd = concat($pgm,' comp sys -n ',${RDA.T_NODE},' -p crs -verbose')
 write '---+ Oracle Clusterware Information Check'
 write '---## Using: ',encode($cmd)
 call writeCommand($cmd)
 write $TOP
 var $cmd = concat($pgm,' comp sys -n ',${RDA.T_NODE},' -p database -verbose')
 write '---+ Oracle Real Application Cluster Check'
 write '---## Using: ',encode($cmd)
 call writeCommand($cmd)
 write $TOP
 var $cmd = concat($pgm,' stage -post crsinst -n ',${RDA.T_NODE},' -verbose')
 write '---+ Oracle Clusterware Post Installation Check'
 write '---## Using: ',encode($cmd)
 call writeCommand($cmd)
 write $TOP
 var $cmd = concat($pgm,' comp software -verbose -n all')
 write '---+ Oracle Clusterware Software Distribution Check'
 write '---## Using: ',encode($cmd)
 call writeCommand($cmd)
 write $TOP
 toc '2:[[',getFile(),'][rda_report][Cluster Verify Information]]'
}

=head2 product_info - Product Information

Provides the inventory of the Cluster Ready Services home if inventory
information is still available. It includes the content of the home directory.

=head2 crs_stat - CRS Stat Details

If Cluster Ready Services is installed, the report provides the detailed output
of the C<crs_stat -u> command.

=cut

if $CRS_HOME
{debug ' Inside CRS module, providing the crs inventory (can take time)'
 report product_info
 title '---+!! Cluster Ready Services Product Information'
 title '---## From ',encode($CRS_HOME),' '
 title $TOC
 call inventory_details(catDir($CRS_HOME,'inventory'),${B_INTERIM})
 prefix
  write '---+ Content of the Cluster Ready Services Home'
 call statDir('n',$CRS_HOME)
 if hasOutput(true)
  write $TOP
 if isCreated(true)
  toc '2:[[',getFile(),'][rda_report][Product Information]]'

 report crs_stat
 if or($R112,$R12)
 {# Use crsctl on 11.2
  debug ' Inside CRS module, providing resource status and configuration'
  var $ttl = '---+!! Status and Configuation Information of Resources'

  prefix
  {if !isCreated()
   {write $ttl
    write $TOC
   }
   write '---+ Status Information of Resources'
   write '---## Using: crsctl status resource -t'
  }
  call writeCommand(concat($CRSCTL,' status resource -t'))
  if hasOutput(true)
   write $TOP

  prefix
  {if !isCreated()
   {write $ttl
    write $TOC
   }
   write '---+ Configuration Information of Resources'
   write '---## Using: crsctl status resource -f'
  }
  call writeCommand(concat($CRSCTL,' status resource -f'))
  if hasOutput(true)
   write $TOP
 }
 if !isCreated(true)
 {debug ' Inside CRS module, providing the crs_stat detailed output'
  var $ttl = '---+!! Cluster Ready Services crs_stat'
  if isVms()
   var $pgm = cond(testFile('f',catFile($ORACLE_HOME,'bin','crs_stat.exe')),\
                   'crs_stat','$!')
  else
   var $pgm = catCommand($CRS_HOME,'bin','crs_stat')

  prefix
  {if !isCreated()
   {write $ttl
    write $TOC
   }
   write '---+ Using: crs_stat -u'
  }
  call writeCommand(concat($pgm,' -u'))
  if hasOutput()
   write $TOP

  prefix
  {if !isCreated()
   {write $ttl
    write $TOC
   }
   write '---+ Using: crs_stat -p'
  }
  call writeCommand(concat($pgm,' -p'))
  if hasOutput(true)
   write $TOP
 }
 if isCreated(true)
  toc '2:[[',getFile(),'][rda_report][CRS Stat Details]]'
}

=head2 Grid Plug and Play

Collects the Grid Plug and Play (GPNP) configuration information.

=cut

pretoc '2:Grid Plug and Play'
debug ' Inside CRS module, gathering GPNP configuration files'
call cat_report(catDir(\
  $CRS_HOME,'gpnp',${RDA.T_NODE},'profiles','peer'),'profile.xml','gpnp_')
unpretoc

=head2 alert_log - Alert Log

Collects the last lines of
F<$CRS_HOME/log/E<lt>nodeE<gt>/alertE<lt>nodeE<gt>.log> (Oracle Database
10g R2 and later).

=head2 crs_log - CRS Log

Handles the log files in F<$CRS_HOME/crs/log> (Oracle Database 10g R1) or
in F<$CRS_HOME/log/E<lt>nodeE<gt>/crsd> (Oracle Database 10g R2 and later).

=head2 css_log - CSS Log

Handles the log files in F<$CRS_HOME/css/log> (Oracle Database 10g R1) or
in F<$CRS_HOME/log/E<lt>nodeE<gt>/cssd> (Oracle Database 10g R2 and later).

=head2 racg_dump - RACG Dump

Gets dump files from F<$CRS_HOME/racg/dump> (Oracle Database 10g R1) or from
F<$CRS_HOME/log/E<lt>nodeE<gt>/racg> and
F<$ORACLE_HOME/log/E<lt>nodeE<gt>/racg> (Oracle Database 10g R2 and later).

=head2 evm_log - CRS EVM Log

Handles the log files in F<$CRS_HOME/evm/log> (Oracle Database 10g R1) or
in F<$CRS_HOME/log/E<lt>nodeE<gt>/evmd> (Oracle Database 10g R2 and later).

=head2 client_log - Client Log

Handles the log files in F<$CRS_HOME/log/E<lt>nodeE<gt>/client> and
F<$ORACLE_HOME/log/E<lt>nodeE<gt>/client> (Oracle Database 10g R2 and later).

=head2 admin_log - Admin Log

Handles the log files in F<$CRS_HOME/log/E<lt>nodeE<gt>/admin> (Oracle
Database 10g R2 and later).

=head2 ohasd_log - Oracle High Availability Service Daemon Logs

Collects the Oracle high availability service daemon log
F<$CRS_HOME/log/E<lt>nodeE<gt>/ohasd/ohasd.log>.
It also handles the Agent Log in
F<$CRS_HOME/log/E<lt>nodeE<gt>/agent/ohasd> and
F<$CRS_HOME/log/E<lt>nodeE<gt>/agent/crsd> directories

=head2 gpnpd_log - Grid Plug and Play Daemon Log

Collects the Grid Plug and Play (GPNP) daemon log
F<$CRS_HOME/log/E<lt>nodeE<gt>/gpnpd/gpnpd.log>.

=head2 lastgasp - Last Gasp Files

Collects the last gasp files from F</etc/oracle/lastgasp>,
F</var/opt/oracle/lastgasp>, F</etc/oracle/oprocd>, and
F</var/opt/oracle/oprocd> directories.

=cut

if $CRS_HOME
{toc '2:CRS Log Files'
 var $log = catDir($CRS_HOME,'log',${RDA.T_NODE})

 debug ' Inside CRS module, gathering alert log file'
 report alert_log
 call tail_file($log,concat('alert',${RDA.T_NODE},'.log'),$LOG_TAIL)
 if isCreated()
  toc '3:[[',getFile(),'][rda_report][Alert Log]]'

 debug ' Inside CRS module, gathering CRS log files'
 report crs_log
 prefix
 {write '---+!! CRS Log Files'
  write $TOC
 }
 var $dir = catDir($CRS_HOME,'crs','log')
 loop $nam (grepDir($dir,'\.log$',$PAT1))
  call tail_file($dir,$nam,$LOG_TAIL,$TOP)
 loop $fil (grepDir(catDir($log,'crsd'),'.',$PAT2))
 {if ?testFile('f',$fil)
  {var $nam = basename($fil)
   next match($nam,$CORE)
   call tail_file(dirname($fil),$nam,$LOG_TAIL,$TOP)
  }
 }
 if isCreated()
  toc '3:[[',getFile(),'][rda_report][CRS Log]]'

 debug ' Inside CRS module, gathering CSS log files'
 report css_log
 prefix
 {write '---+!! CSS Log Files'
  write $TOC
 }
 var $dir = catDir($CRS_HOME,'css','log')
 loop $nam (grepDir($dir,'\.log$',$PAT1))
  call tail_file($dir,$nam,$LOG_TAIL,$TOP)
 var $dir = catDir($CRS_HOME,'css','init')
 loop $nam (grepDir($dir,'\.log$',$PAT1))
  call tail_file($dir,$nam,$LOG_TAIL,$TOP)
 loop $fil (grepDir(catDir($log,'cssd'),'.',$PAT2))
 {if ?testFile('f',$fil)
  {var $nam = basename($fil)
   next match($nam,$CORE)
   call tail_file(dirname($fil),$nam,$LOG_TAIL,$TOP)
  }
 }
 if isCreated()
  toc '3:[[',getFile(),'][rda_report][CSS Log]]'

 debug ' Inside CRS module, gathering RACG dump files'
 report racg_dump
 prefix
 {write '---+!! RACG dump Files'
  write $TOC
 }
 var $dir = catDir($CRS_HOME,'racg','dump')
 loop $nam (grepDir($dir,'.*',$PAT1))
  call tail_file($dir,$nam,$CRS_TAIL,$TOP)
 var $alt = catDir($ORACLE_HOME,'racg','dump')
 if !sameDir($alt,$dir)
 {loop $nam (grepDir($alt,'.*',$PAT1))
  {next match($nam,$CORE)
   call tail_file($alt,$nam,$CRS_TAIL,$TOP)
  }
 }
 var $dir = catDir($log,'racg')
 loop $fil (grepDir($dir,'.',$PAT2))
 {if ?testFile('f',$fil)
  {var $nam = basename($fil)
   next match($nam,$CORE)
   call tail_file(dirname($fil),$nam,$LOG_TAIL,$TOP)
  }
 }
 var $alt = catDir($ORACLE_HOME,'log',${RDA.T_NODE},'racg')
 if !sameDir($alt,$dir)
 {loop $fil (grepDir($alt,'.',$PAT2))
  {if ?testFile('f',$fil)
   {var $nam = basename($fil)
    next match($nam,$CORE)
    call tail_file(dirname($fil),$nam,$LOG_TAIL,$TOP)
   }
  }
 }
 if isCreated()
  toc '3:[[',getFile(),'][rda_report][RACG Dump]]'

 debug ' Inside CRS module, gathering CRS EVM Log'
 report evm_log
 prefix
 {write '---+!! CRS EVM Files'
  write $TOC
 }
 var $dir = catDir($CRS_HOME,'evm','log')
 loop $nam (grepDir($dir,'\.log$',$PAT1))
  call tail_file($dir,$nam,$LOG_TAIL,$TOP)
 var $dir = catDir($CRS_HOME,'evm','init')
 loop $nam (grepDir($dir,'\.log$',$PAT1))
  call tail_file($dir,$nam,$LOG_TAIL,$TOP)
 loop $fil (grepDir(catDir($log,'evmd'),'.',$PAT2))
 {if ?testFile('f',$fil)
  {var $nam = basename($fil)
   next match($nam,$CORE)
   call tail_file(dirname($fil),$nam,$LOG_TAIL,$TOP)
  }
 }
 if isCreated(true)
  toc '3:[[',getFile(),'][rda_report][CRS EVM Log]]'

 debug ' Inside CRS module, gathering Client Log'
 report client_log
 prefix
 {write '---+!! Client Log Files'
  write $TOC
 }
 var $dir = catDir($log,'client')
 loop $fil (grepDir($dir,'.',$PAT2))
 {if ?testFile('f',$fil)
  {var $nam = basename($fil)
   next match($nam,$CORE)
   call tail_file(dirname($fil),$nam,$LOG_TAIL,$TOP)
  }
 }
 var $alt = catDir($ORACLE_HOME,'log',${RDA.T_NODE},'client')
 if !sameDir($alt,$dir)
 {loop $fil (grepDir($alt,'.',$PAT2))
  {if ?testFile('f',$fil)
   {var $nam = basename($fil)
    next match($nam,$CORE)
    call tail_file(dirname($fil),$nam,$LOG_TAIL,$TOP)
   }
  }
 }
 if isCreated(true)
  toc '3:[[',getFile(),'][rda_report][Client Log]]'

 debug ' Inside CRS module, gathering Admin Log'
 report admin_log
 prefix
 {write '---+!! Admin Log Files'
  write $TOC
 }
 loop $fil (grepDir(catDir($log,'admin'),'.',$PAT2))
 {if ?testFile('f',$fil)
  {var $nam = basename($fil)
   next match($nam,$CORE)
   call tail_file(dirname($fil),$nam,$LOG_TAIL,$TOP)
  }
 }
 if isCreated(true)
  toc '3:[[',getFile(),'][rda_report][Admin Log]]'

 debug ' Inside CRS module, gathering OHASD logs'
 report ohasd_log
 var $ttl = '---+!! Oracle High Availability Service Daemon Logs'
 var $fil = catFile($log,'ohasd','ohasd.log')
 if ?testFile('f',$fil)
 {var $siz = getSize($fil)
  var $lnk = encode($fil)
  if $siz
  {output d,'ohasd.log'
   if ${CUR.O_LAST}->write_data($fil)
    var $lnk = concat('[[',${CUR.O_LAST}->get_raw(true),'][_blank][',$lnk,']]')
   end ${CUR.O_LAST}
  }
  write $ttl
  write $TOC
  write $WARN
  write '---++ OHASD Log'
  write '|*File Name*| *Size*|*Last Modified Date*|'
  write '|',$lnk,' | ',$siz,'|',getLastModify($fil,'<UTC>'),' |'
  write $TOP
 }
 prefix
 {if !isCreated()
  {write $ttl
   write $TOC
   write $WARN
  }
  write '---++ Agent Logs'
  write '|*File Name*| *Size*|*Last Modified Date*|'
 }
 loop $dir (grepDir(catDir($log,'agent'),'^(ohasd|crsd)$','ip'))
 {loop $sub (grepDir($dir,'^\.+$','nv'))
  {var $fil = catFile($dir,$sub,concat($sub,'.log'))
   if ?testFile('f',$fil)
   {var $siz = getSize($fil)
    var $lnk = encode($fil)
    if $siz
    {output d,concat(basename($fil),'.log')
     if ${CUR.O_LAST}->write_data($fil)
      var $lnk = concat('[[',${CUR.O_LAST}->get_raw(true),\
                        '][_blank][',$lnk,']]')
     end ${CUR.O_LAST}
    }
    write '|',$lnk,' | ',$siz,'|',getLastModify($fil,'<UTC>'),' |'
   }
  }
 }
 if isCreated(true)
  toc '3:[[',getFile(),\
      '][rda_report][Oracle High Availability Service Daemon Logs]]'

 if ?testFile('f',$fil = catFile($log,'gpnpd','gpnpd.log'))
 {debug ' Inside CRS module, gathering GPNPD log'
  report gpnpd_log
  var $siz = getSize($fil)
  var $lnk = encode($fil)
  if $siz
  {output d,'gpnpd.log'
   if ${CUR.O_LAST}->write_data($fil)
    var $lnk = concat('[[',${CUR.O_LAST}->get_raw(true),'][_blank][',$lnk,']]')
   end ${CUR.O_LAST}
  }
  write '---+ GPNPD Log'
  write $WARN
  write '|*File Name*| *Size*|*Last Modified Date*|'
  write '|',$lnk,' | ',$siz,'|',getLastModify($fil,'<UTC>'),' |'
  toc '3:[[',getFile(),'][rda_report][Grid Plug and Play Daemon Log]]'
 }

 debug ' Inside CRS module, gathering last gasp files'
 report lastgasp
 prefix
 {write '---+!! Last Gasp Files'
  write $WARN
  write '   * Last Gasp Files (lgl) files are collected for the last 30 days \
              and when their size exceeds 4096 bytes.'
  write '|*File Name*| *Size*|*Last Modified Date*|'
 }
 loop $dir ('/etc/oracle/oprocd','/var/opt/oracle/oprocd')
 {loop $fil (grepDir($dir,'\.oprocd\.log$','inp'))
  {var $siz = getSize($fil)
   var $lnk = encode($fil)
   if $siz
   {output d,basename($fil)
    if ${CUR.O_LAST}->write_data($fil)
     var $lnk = concat('[[',${CUR.O_LAST}->get_raw(true),'][_blank][',$lnk,']]')
    end ${CUR.O_LAST}
   }
   write '|',$lnk,' | ',$siz,'|',getLastModify($fil,'<UTC>'),' |'
  }
 }
 loop $dir ('/etc/oracle/lastgasp','/var/opt/oracle/lastgasp')
 {loop $fil (grepDir($dir,'^(css)?(mo|ag).*\.lgl$','inp'))
  {var $siz = getSize($fil)
   var $lnk = encode($fil)
   if and(not(isFiltered()),expr('>',$siz,4096),isNewer($fil,30))
   {output b,basename($fil)
    if ${CUR.O_LAST}->write_data($fil)
     var $lnk = concat('[[',${CUR.O_LAST}->get_raw(true),'][_blank][',$lnk,']]')
    end ${CUR.O_LAST}
   }
   write '|',$lnk,' | ',$siz,'|',getLastModify($fil,'<UTC>'),' |'
  }
 }
 if isCreated(true)
  toc '3:[[',getFile(),'][rda_report][Last Gasp Files]]'

=head2 Other Log Files

Collects log files from Grid base directory.

=cut

 debug ' Inside CRS module, gathering other log files'
 pretoc '3:Other Log Files'
 if isVms()
  var $pgm = 'orabase'
 else
 {var $pgm = catCommand($CRS_HOME,'bin',${AS.EXE:'orabase'})
  var $env = setLocalEnv('ORACLE_HOME',$CRS_HOME)
 }
 if grepCommand($pgm,'^(.+)$','f1')
  call sort_files(4,0,grepDir(first,'\.log$',$PAT2))
 if !isVms()
  call setLocalEnv('ORACLE_HOME',$env)
 unpretoc

=for stopwords ocrdump

=head2 ocrdump - ocrdump

You can request an OCR dump by assigning a true (nonzero) value to the
C<DB.CRS.B_OCRDUMP> setting.

=for stopwords olrdump

=head2 olrdump - olrdump

Gathers Oracle Cluster/Local Registry dump.

=cut

 collect DB:MCcrs|CRS($CRS_HOME)

=for stopwords ocrcheck

=head2 ocrcheck - ocrcheck

Executes C<ocrcheck> (Oracle Database 10g R2 and later).

=cut

 debug ' Inside CRS module, requesting an OCR check'
 report ocrcheck
 if isVms()
  var $pgm = 'ocrcheck'
 else
 {var $pgm = catCommand($CRS_HOME,'bin','ocrcheck')
  var $env = setLocalEnv('ORACLE_HOME',$CRS_HOME)
 }
 prefix
 {write '---+ OCR Check'
  write '---## Using: ',encode($pgm)
 }
 call writeCommand($pgm)
 if isCreated(true)
  toc '3:[[',getFile(),'][rda_report][ocrcheck]]'
 if !isVms()
  call setLocalEnv('ORACLE_HOME',$env)

=for stopwords ocrconfig showbackup

=head2 ocrconfig - ocrconfig -showbackup

Executes C<ocrconfig -showbackup> (Oracle Database 10g R2 and later).

=cut

 debug ' Inside CRS module, requesting an OCR config'
 report ocrconfig
 if isVms()
  var $pgm = 'ocrconfig -showbackup'
 else
  var $pgm = concat(catCommand($CRS_HOME,'bin','ocrconfig'),' -showbackup')
 prefix
 {write '---+ OCR Config'
  write '---## Using: ',encode($pgm)
 }
 call writeCommand($pgm)
 if isCreated(true)
  toc '3:[[',getFile(),'][rda_report][ocrconfig -showbackup]]'

=head2 Installation Files

Collects the contents of the installation files.

=cut

 debug ' Inside CRS module, getting installation files content'
 pretoc '2:Installation Files'
 call sort_files(3,0,\
   grepDir(catDir($CRS_HOME,'cfgtoollogs','opatch'),'\.log$',$PAT2),\
   grepDir(catDir($CRS_HOME,'.patch_storage'),'^\.+$',concat('drvm',$AGE)))
 unpretoc
}

=head2 srvctl - SRVM

Runs F<srvctl> commands and reports the configuration information (Oracle
Database 9i).

=cut

=head2 srvctl - SRVCTL

Runs F<srvctl> commands and reports the configuration information (Oracle
Database 10g and later).

=cut

debug ' Inside CRS module, running srvctl commands'
report srvctl
write '---+!! SRVCTL Information'
write $TOC

var $R9 = testDir('d',catDir($ORACLE_HOME,'oracm'))
if $R9
{write '---+ Location and Contents of srvConfig.loc'
 if !?$loc = nvl(testFile('f','/var/opt/oracle/srvConfig.loc'),\
                 testFile('f','/etc/srvConfig.loc'))
  write '**srvConfig.loc not found.**'
 else
 {write '---++ Location'
  call statFile('p',$loc)
  write $TOP

  write '---++ Contents of ',encode($loc)
  call writeFile($loc)
  write $TOP

  write '---++ srvConfig File Permissions'
  loop $fil (grepFile($loc,'srvconfig_loc'))
  {var @fld = split('=',trim($fil))
   call statFile('p',$fld[-1])
  }
  write $TOP
 }
}

if or(and($CRS_HOME,\
          testFile($MOD,catFile($CRS_HOME,'bin',${AS.BATCH:'srvctl'}))),\
      testFile($MOD,catFile($ORACLE_HOME,'bin',${AS.BATCH:'srvctl'})))
{var $pgm = cond(isVms(),'srvctl',lastCommand())
 if grepCommand(concat($pgm,' -V'),'(11\.2|12\.[12])','f')
 {write '---+ Nodeapps Configuration Information'
  write '---## Using: srvctl config nodeapps'
  if isVms()
   call writeCommand('srvctl config nodeapps')
  else
   call writeCommand(concat($pgm,' config nodeapps'))
  write $TOP

  write '---+ Database Configuration Information'
  if isVms()
   call loadCommand('srvctl config')
  else
   call loadCommand(concat($pgm,' config'))
  loop $db (getLines())
  {break match($db,'^PRKH-\d+\s*:')
   if length($db)
   {write '---++ ',$db,' Database'
    var $db = quote($db)
    write '---+++ Database Configuration'
    write '---### Using: srvctl config database -d ',$db,' -a'
    if isVms()
     call writeCommand(concat('srvctl config database "-d" ',$db,' "-a"'))
    else
     call writeCommand(concat($pgm,' config database -d ',$db,' -a'))

    write '---+++ Database Services'
    write '---### Using: srvctl config service -d ',$db,' -v'
    if isVms()
     call writeCommand(concat('srvctl config service "-d" ',$db,' "-v"'))
    else
     call writeCommand(concat($pgm,' config service -d ',$db,' -v'))
   }
  }
  write $TOP

  write '---+ Configuration Information for Scan Listener'
  write '---## Using: srvctl config scan_listener'
  call writeCommand(concat($pgm,' config scan_listener'))
  write $TOP

  write '---+ Current State for Scan Listener'
  write '---## Using: srvctl status scan_listener'
  call writeCommand(concat($pgm,' status scan_listener'))
  write $TOP

  write '---+ Configuration for Scan VIP'
  write '---## Using: srvctl config scan'
  call writeCommand(concat($pgm,' config scan'))
  write $TOP

  write '---+ Current State for Scan VIP'
  write '---## Using: srvctl status scan'
  call writeCommand(concat($pgm,' status scan'))
  write $TOP

  write '---+ Configuration for Server Pool'
  write '---## Using: srvctl config srvpool'
  call writeCommand(concat($pgm,' config srvpool'))
  write $TOP

  write '---+ Current State for Server Pool'
  write '---## Using: srvctl status srvpool -a'
  if isVms()
   call writeCommand('srvctl status srvpool "-a"')
  else
   call writeCommand(concat($pgm,' status srvpool -a'))
  write $TOP

  write '---+ Configured Listeners Registered with Oracle Clusterware'
  write '---## Using: srvctl config listener'
  call writeCommand(concat($pgm,' config listener'))
  write $TOP

  write '---+ Status of Listener Resources'
  write '---## Using: srvctl status listener'
  call writeCommand(concat($pgm,' status listener'))
  write $TOP

  write '---+ ASM Configuration'
  write '---## Using: srvctl config asm -a'
  if isVms()
   call writeCommand('srvctl config asm "-a"')
  else
   call writeCommand(concat($pgm,' config asm -a'))
  write $TOP

  write '---+ GNS Daemon Configuration'
  write '---## Using: srvctl config gns -a'
  if isVms()
   call writeCommand('srvctl config gns "-a"')
  else
   call writeCommand(concat($pgm,' config gns -a'))
  write $TOP
 }
 else
 {write '---+ Nodeapps Configuration Information'
  write '---## Using: srvctl config nodeapps -n ',${RDA.T_NODE},' -a -g -s -l'
  if isVms()
   call writeCommand(concat('srvctl config nodeapps "-n" ',${RDA.T_NODE},\
                            ' "-a" "-g" "-s" "-l"'))
  else
   call writeCommand(concat($pgm,' config nodeapps -n ',${RDA.T_NODE},\
                            ' -a -g -s -l'))
  write $TOP

  write '---+ Database Configuration Information'
  if isVms()
   call loadCommand('srvctl config')
  else
   call loadCommand(concat($pgm,' config'))
  if $R9
  {write '---## Using: srvctl config'
   call writeLastFile()
  }
  else
  {loop $db (getLines())
   {break match($db,'^PRKH-\d+\s*:')
    if length($db)
    {write '---++ ',$db,' Database'
     var $db = quote($db)
     write '---### Using: srvctl config database -d ',$db,' -a'
     if isVms()
      call writeCommand(concat('srvctl config database "-d" ',$db,' "-a"'))
     else
      call writeCommand(concat($pgm,' config database -d ',$db,' -a'))
    }
   }
  }
  write $TOP

  write '---+ Configured Listeners Registered with Oracle Clusterware'
  write '---## Using: srvctl config listener -n ',${RDA.T_NODE}
  if isVms()
   call writeCommand(concat('srvctl config listener "-n" "',${RDA.T_NODE},'"'))
  else
   call writeCommand(concat($pgm,' config listener -n ',${RDA.T_NODE}))
  write $TOP
 }
}
if $R9
 toc '2:[[',getFile(),'][rda_report][SRVM]]'
else
 toc '2:[[',getFile(),'][rda_report][SRVCTL]]'

=head2 oifcfg - Interfaces

Collects interface information using C<oifcfg>.

=cut

debug ' Inside CRS module, gathering interface information'
report oifcfg
var $TTL = '---+!! Interface Information'

prefix
{write '---+ Available Interfaces'
 write '---## Using: oifcfg iflist'
}
if isVms()
 call writeCommand('oifcfg_ifls')
else
 call writeCommand(concat(catCommand($CRS_HOME,'bin','oifcfg'),' iflist'))
if hasOutput(true)
 write $TOP

prefix
{write '---+ Defined Interfaces'
 write '---## Using: oifcfg getif'
}
if isVms()
 call writeCommand('oifcfg getif')
else
 call writeCommand(concat(catCommand($CRS_HOME,'bin','oifcfg'),' getif'))
if hasOutput(true)
 write $TOP

if isCreated(true)
 toc '2:[[',getFile(),'][rda_report][Interfaces]]'

=head2 interconnect - Interconnect

Collects interconnect information from the database. It requires SYSDBA
privileges.

=cut

debug ' Inside CRS module, gathering interconnect information'
report interconnect
var $TTL = '---+!! Interconnect Information'
var @TTL = ('',\
            '---+ x$ksxpia Information',\
            '---+ x$sksxpia Information')
var @HDR = ()
var ($HDR[1],$col1) = getSqlColumns('RDA','','X$KSXPIA')
call clearSqlColumns('RDA')
var ($HDR[2],$col2) = getSqlColumns('RDA','','X$SKGXPIA')
call clearSqlColumns('RDA')
set $sql
{SELECT :1
" FROM x$ksxpia;
"PROMPT ___Macro_separator(2)___
"SELECT :2
" FROM x$skgxpia;
}
call separator(1)
call writeSql(bindSql($sql,$col1,$col2))
call separator(0,'Interconnect')

=head2 ipc - IPC

Runs C<oradebug> commands to generate a trace file, and then displays the IPC
trace file. It is performed for databases associated with the current Oracle
home only. RDA does not execute the command for Windows.

=cut

if !or(isWindows(),isCygwin())
{debug ' Inside CRS module, gathering IPC trace file'
 report ipc

 prefix
  write '---+ IPC Trace File'
 if ${I_DBC}->get_first('B_LOCAL')
 {set $sql
  {oradebug setmypid
  "oradebug ipc
  "oradebug tracefile_name
  }
  call loadSql($sql)
  var ($fil) = grepLastSql('\.trc')
  if $fil
   call writeFile(catFile($fil))
  else
  {write '<verbatim>'
   call writeLastSql()
   if getSqlMessage()
    write last
   write '</verbatim>'
  }
 }
 else
  write 'Database was not chosen as an option for this execution.%BR%\
         oradebug ipc information not gathered.'
 if isCreated(true)
  toc '2:[[',getFile(),'][rda_report][IPC]]'
}

=head2 logs - Cluster Configuration/Logs

Looks for the cluster configuration and log files on the operating system. The
file list is platform-specific (Oracle Database 9i).

=cut

if $R9
{debug ' Inside CRS module, looking for cluster configuration and logs'
 report logs
 prefix
 {write '---+!! Cluster Configuration and Logs'
  write $TOC
 }
 loop $fil (@CFG)
 {if ?testFile('f',$fil)
   call cat_file(dirname($fil),basename($fil),100)
  else
   write '---## ',encode($fil),' not present on this server'
  write $TOP
 }
 loop $fil (@LOG)
 {if ?testFile('f',$fil)
   call tail_file(dirname($fil),basename($fil),100)
  else
   write '---## ',encode($fil),' not present on this server'
  write $TOP
 }
 if isCreated(true)
  toc '2:[[',getFile(),'][rda_report][Cluster Configuration/Logs]]'
}

# Disable the group title in next index
toc '-:Cluster'

=head1 SEE ALSO

L<DB:DBinfo|collect::DB:DBinfo>,
L<DB:CRSinit|collect::DB:CRSinit>,
L<DB:MCcrs|collect::DB:MCcrs>,
L<OS:COREinfo|collect::OS:COREinfo>,
L<OS:OSaix|collect::OS:OSaix>,
L<OS:OShpux|collect::OS:OShpux>,
L<OS:OSlinux|collect::OS:OSlinux>,
L<OS:OSosf|collect::OS:OSosf>,
L<OS:OSsunos|collect::OS:OSsunos>,
L<OS:OSwin32|collect::OS:OSwin32>
L<RDA:INVinfo|collect::RDA:INVinfo>,
L<RDA:library|collect::RDA:library>

=begin credits

=over 10

=item RDA 4.0:  Bernhard Cock Buning.

=item RDA 4.2:  Roger Snowden.

=item RDA 4.6:  Bob Caldwell.

=item RDA 4.15: Jaime Alcoreza, Grant Hayden, Hagen Herbst, Scott Jesse.

=item RDA 4.18: Jaime Alcoreza.

=item RDA 4.19: Jaime Alcoreza.

=item RDA 4.20: Grant Hayden, Takeyoshi Sasaki.

=item RDA 4.21: Jaime Alcoreza.

=item RDA 4.22: Jaime Alcoreza, Grant Hayden.

=item RDA 4.24: Jaime Alcoreza, Hagen Herbst.

=item RDA 4.27: Ericka Washington.

=item RDA 8.00: Jamie Alcoreza.

=item RDA 8.02: Jamie Alcoreza.

=item RDA 8.03: Jamie Alcoreza.

=back

=end credits

=head1 COPYRIGHT NOTICE

Copyright (c) 2002, 2016, Oracle and/or its affiliates. All rights reserved.

=head1 TRADEMARK NOTICE

Oracle and Java are registered trademarks of Oracle and/or its
affiliates. Other names may be trademarks of their respective owners.

=cut