Oracle Install the RAC Database 10g for HP-UX with ASM Disk (EVA storage)

** Myself




Host Rac Information DB ONLINE
mkondb1 information :
- hostname= mkondb1
- virtual ip(for rac) mkondb1-vip=172.16.10.15
- public ip1(for remote) mkondb1-pub1=172.16.10.13
- public ip2(for remote) mkondb1-pub2=172.16.10.14
- private ip1 (for heartbeat) mkondb1-priv1=192.168.1.13
- private ip2 (for heartbeat) mkondb1-priv2=192.168.1.14
- oracle sid = ONLINE1
mkondb2 information :
- hostname= mkondb2
- virtual ip(for rac) mkondb2-vip=172.16.10.18
- public ip1(for remote) mkondb2-pub1=172.16.10.16
- public ip2(for remote) mkondb2-pub2=172.16.10.17
- private ip1 (for heartbeat) mkondb2-priv1=192.168.1.16
- private ip2 (for heartbeat) mkondb2-priv2=192.168.1.17
- oracle sid = ONLINE2

Database Rac Information
ONLINE Database
- Instance= ONLINE1, ONLINE2
- DB storage-->Raw device=+ASM
- +ASM instance=+ASM1,+ASM2
- +ASM storage files= control01.ctl, control02.ctl,control03.ctl,system01.dbf, sysaux01.dbf, temp01.dbf, undotbs01.dbf,undotbs02.dbf
- Ocr --> raw= /dev/rdisk/disk32
- Votting --> raw= /dev/rdisk/disk33
- Archivelog-->
ON Node1=/oracle/archive/ONLINE1
ON Node2=/oracle/archive/ONLINE2
- Spfille-->ASM= +DATA
ONLINE1 :Instance 1 or Thread 1
- hostname=mkondb1
- oracle sid=ONLINE1
- oracle cluster sw=/oracle/product/crs
- oracle asm sw=/oracle/product/asm
- oracle databse sw=/oracle/product/db_1
redolog for instance1 on shared storage +ASM1
undo for instance1 on shared storage +ASM1 = undotbs01.dbf
ONLINE2 :Instance 2 or Thread 2
- hostname=mkondb2
- oracle sid=ONLINE2
- oracle cluster sw=/oracle/product/crs
- oracle asm sw=/oracle/product/
- oracle databse sw=/oracle/product/db_1
redolog for instance2 on shared storage +ASM2
undo for instance2 on shared storage +ASM2 = undotbs02.dbf
Installation Rac ONLINE DATABASE
Step of Installation RAC ONLINE

1. Configure network interface and resolve hostname(DNS or hostfile)
2. Create shared disks raw& shared disk filesystem.
3. Install oracle clusterware software.
4. Install the Database Software and Create an ASM Instance.
5. Create a Database using DBCA
6. TNS and Listener configuration
7. Enable Archive log file and Shared NFS

1. Configure network interface and resolve hostname

/etc/host
172.16.10.13 mkondb1
172.16.10.15 mkondb1-vip
10.10.9.13 mkondb1-priv
172.16.10.14 mkondb1-pub2
192.168.1.14 mkondb1-priv2
172.16.10.16 mkondb2
172.16.10.18 mkondb2-vip
10.10.9.17 mkondb2-priv
172.16.10.17 mkondb2-pub2
192.168.1.17 mkondb2-priv2
127.0.0.1 localhost loopback
kctune
Tunable Value Expression Changes
ksi_alloc_max (now) 33600 Default Immed
(next boot) 33600 nproc*8
maxssiz (now) 8388608 Default Immed
(next boot) 134217728 134217728
maxssiz_64bit (now) 268435456 Default Immed
(next boot) 1073741824 1073741824
maxuprc (now) 256 Default Immed
(next boot) 3780 ((nproc*9)/10)
msgmni (now) 512 Default Immed
(next boot) 4200 nproc
msgtql (now) 1024 Default Immed
(next boot) 4200 nproc
ncsize (now) 8976 Default
(next boot) 36672 36672
nflocks (now) 4096 Default Auto
(next boot) 4200 nproc
ninode (now) 8192 Default
(next boot) 35648 (8*nproc+2048)
nkthread (now) 8416 Default Immed
(next boot) 8416 8416
nproc 4200 Default Immed
semmni (now) 2048 Default
(next boot) 4200 nproc
semmns (now) 4096 Default
(next boot) 8400 semmni*2
semmnu (now) 256 Default
(next boot) 4196 nproc-4
shmmax (now) 1073741824 Default Immed
(next boot) 4294967296 4294967296
shmmni (now) 400 Default Immed
(next boot) 512 512
vps_ceiling (now) 16 Default Immed
(next boot) 64 64





2. Create shared disks raw& shared disk filesystem


-Create disk from storage EVA


-Lun design
oracle_arch_mkondb1 100GB
oracle_arch_mkondb2 100GB
oracle_shared_prd_ocr 2GB
oracle_shared_prd_data1 200GB
oracle_shared_prd_data2 200GB
oracle_shared_prd_vote 1GB
oracle_sw_mkondb1 20GB
oracle_sw_mkondb2 20GB
-OS disk device on Node1 and Node2
vgoraarch-->lvoraarch-->/dev/disk/disk34 -->oracle_arch_mkondb1
vgorasw-->lvoraswcrs (4G), lvoraswasm(8G), lvoraswdb(8G) -->/dev/disk/disk29 --> oracle_sw_mkondb1
raw ocr -->/dev/rdisk/disk32--> oracle_shared_prd_ocr
raw vote -->/dev/rdisk/disk33--> oracle_shared_prd_vote
raw data(ASM disk) -->/dev/rdisk/disk40--> oracle_shared_prd_data1
raw data(ASM disk) -->/dev/rdisk/disk41--> oracle_shared_prd_data2
change disk id 2 machine
/tmp/infile
64000/0xfa00/0xe disk 31
ioinit -f /tmp/infile
rmsf /dev/disk/disk21
rmsf /dev/rdisk/disk21
Create vg,lv,file, for SW ora,crs 2 node
pvcreate /dev/rdisk/disk29
mkdir /dev/vgorasw
mknod /dev/vgorasw/group c 64 0x010000
vgcreate -s 32 /dev/vgorasw /dev/disk/disk29
lvcreate -L 4096 -n lvoraswcrs /dev/vgorasw
lvcreate –L 8192 -n lvoraswasm /dev/vgorasw
lvcreate -l 223 -n lvoraswdb /dev/vgorasw
mkfs -F vxfs -o largefiles /dev/vgorasw/lvoraswcrs
mkfs -F vxfs -o largefiles /dev/vgorasw/lvoraswasm
mkfs -F vxfs -o largefiles /dev/vgorasw/lvoraswdb
mkdir -p /oracle/product/crs
mkdir -p /oracle/product/asm
mkdir -p /oracle/product/db_1
/etc/fstab
/dev/vgorasw/lvoraswcrs /oracle/product/crs vxfs delaylog 0 2
/dev/vgorasw/lvoraswasm /oracle/product/asm vxfs delaylog 0 2
/dev/vgorasw/lvoraswdb /oracle/product/db_1 vxfs delaylog 0 2
mount –a
Create Archive storage disk
Scan disk
ioscan –fnNC disk
change disk id 2 machine
/tmp/infile
64000/0xfa00/0x11 disk 34
ioinit -f /tmp/infile
rmsf /dev/disk/disk11
rmsf /dev/rdisk/disk11
Create vg,lv,file, for archive 2 node
pvcreate /dev/rdisk/disk34
mkdir /dev/vgoraarch
mknod /dev/vgoraarch/group c 64 0x020000
vgcreate -s 32 /dev/vgoraarch /dev/disk/disk34
lvcreate -L 102368 -n lvoraarch /dev/vgoraarch
mkfs -F vxfs -o largefiles /dev/vgoraarch/lvoraarch
#Node 1
mkdir -p /oracle/archive/ONLINE1
#Node 2
mkdir -p /oracle/archive/ONLINE2
/etc/fstab #Node1
/dev/vgoraarch/lvoraarch /oracle/archive/ONLINE1 vxfs delaylog 0 2
/etc/fstab #Node2
/dev/vgoraarch/lvoraarch /oracle/archive/ONLINE2 vxfs delaylog 0 2
mount –a
chown oracle:oinstall /oracle/archive/ONLINE1
chown oracle:oinstall /oracle/archive/ONLINE2


Create DATA storage disk for ASM
Scan disk
ioscan –fnNC disk

change disk id 2 machine
/tmp/infile
64000/0xfa00/0x16 disk 40
64000/0xfa00/0x17 disk 41
ioscan -m dsf
ioinit -f /tmp/infile
rmsf /dev/disk/disk16
rmsf /dev/rdisk/disk16
rmsf /dev/disk/disk17
rmsf /dev/rdisk/disk17
chown oracle:dba /dev/rdisk/disk40
chmod 660 /dev/rdisk/disk40
chown oracle:dba /dev/rdisk/disk41
chmod 660 /dev/rdisk/disk41












3. Install oracle clusterware software
- Configuration user environment
groupadd -g 201 dba
groupadd -g 200 oinstall
useradd -u 200 -g oinstall -G dba -d /oracle oracle
cp /etc/skel/.profile /oracle/
chown oracle:oinstall /oracle
passwd oracle
# .rhosts file in $HOME of root
echo “+ root” > /.rhosts

# .rhosts file in $HOME of oracle
echo “+ oracle” > /oracle/.rhosts
chmod 700 /.rhosts
chmod 700 /oracle/.rhosts
/etc/privgroup
dba MLOCK RTSCHED RTPRIO
setprivgrp -f /etc/privgroup

chown oracle:dba /dev/async
chmod 660 /dev/async
chown oracle:oinstall /oracle/product
chown -R oracle:oinstall /oracle/product
chmod -R 775 /oracle/product
OCR
chown root:oinstall /dev/rdisk/disk32
chmod 640 /dev/rdisk/disk32
Vote
chown oracle:dba /dev/rdisk/disk33
chmod 660 /dev/rdisk/disk33
###### root profile ########
/.profile
export ORACLE_BASE=/oracle/product
export ORA_CRS_HOME=$ORACLE_BASE/crs
export PATH=$PATH:ORA_CRS_HOME/bin:
print ' '
print '$ORA_CRS_HOME: '$ORA_CRS_HOME
print ' '
# ALIAS
alias psg="ps -ef | grep"
alias lla="ll -rta"
alias crs="cd $ORA_CRS_HOME"
alias crsbin="cd $ORA_CRS_HOME/bin"
###### oracle profile ########
Node1 /oracle/.profile
umask 022
export ORACLE_BASE=/oracle/product
export ORACLE_HOME=$ORACLE_BASE/db_1
export ORA_CRS_HOME=$ORACLE_BASE/crs
export ORA_ASM_HOME=$ORACLE_BASE/asm
export ORACLE_SID=ONLINE1
export ORACLE_TERM=xterm
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib:$ORACLE_HOME/rdbms/lib
export PATH=$PATH:$ORACLE_HOME/bin:$ORA_CRS_HOME/bin:$ORA_ASM_HOME/bin
export CLASSPATH=$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib/:$ORACLE_HOME/network/jlib
print ' '
print '$ORACLE_SID: '$ORACLE_SID
print '$ORACLE_HOME: '$ORACLE_HOME
print '$ORA_CRS_HOME: '$ORA_CRS_HOME
print '$ORA_ASM_HOME: '$ORA_ASM_HOME
print ' '
# ALIAS
alias psg="ps -ef | grep"
alias lla="ll -rta"
alias sq="ied sqlplus '/as sysdba'"
alias oh="cd $ORACLE_HOME"
alias ohbin="cd $ORACLE_HOME/bin"
alias crs="cd $ORA_CRS_HOME"
alias crsbin="cd $ORA_CRS_HOME/bin"
alias asm="cd $ORA_ASM_HOME"
alias asmbin="cd $ORA_ASM_HOME/bin"
Node1 /oracle/.profile
umask 022
export ORACLE_BASE=/oracle/product
export ORACLE_HOME=$ORACLE_BASE/db_1
export ORA_CRS_HOME=$ORACLE_BASE/crs
export ORA_ASM_HOME=$ORACLE_BASE/asm
export ORACLE_SID=ONLINE2
export ORACLE_TERM=xterm
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib:$ORACLE_HOME/rdbms/lib
export PATH=$PATH:$ORACLE_HOME/bin:$ORA_CRS_HOME/bin:$ORA_ASM_HOME/bin
export CLASSPATH=$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib/:$ORACLE_HOME/network/jlib
print ' '
print '$ORACLE_SID: '$ORACLE_SID
print '$ORACLE_HOME: '$ORACLE_HOME
print '$ORA_CRS_HOME: '$ORA_CRS_HOME
print '$ORA_ASM_HOME: '$ORA_ASM_HOME
print ' '
# ALIAS
alias psg="ps -ef | grep"
alias lla="ll -rta"
alias sq="ied sqlplus '/as sysdba'"
alias oh="cd $ORACLE_HOME"
alias ohbin="cd $ORACLE_HOME/bin"
alias crs="cd $ORA_CRS_HOME"
alias crsbin="cd $ORA_CRS_HOME/bin"
alias asm="cd $ORA_ASM_HOME"
alias asmbin="cd $ORA_ASM_HOME/bin"
chown oracle:dba /oracle/.profile
chmod 644 /oracle/.profile

Required if Motif 2.1 Development Environment Package is not installed
ln -s /usr/lib/libX11.3 /usr/lib/libX11.sl
ln -s /usr/lib/libXIE.2 /usr/lib/libXIE.sl
ln -s /usr/lib/libXext.3 /usr/lib/libXext.sl
ln -s /usr/lib/libXhp11.3 /usr/lib/Xhp11.sl
ln -s /usr/lib/libXi.3 /usr/lib/libXi.sl
ln -s /usr/lib/libXm.4 /usr/lib/libXm.sl
ln -s /usr/lib/libXp.2 /usr/lib/libXp.sl
ln -s /usr/lib/libXt.3 /usr/lib/libXt.sl
ln -s /usr/lib/libXtst.2 /usr/lib/libXtst.sl
ssh root,oracle
ssh-keygen -t dsa
cp /.ssh/id_dsa.pub /.ssh/authorized_keys
- Install Oracle CRS 10.2.0.1
./runInstaller -ignoreSysPrereqs

CRS config
mkondedb1 mkondedb1-priv1 mkondedb1-vip
mkondedb2 mkondedb2-priv1 mkondedb2-vip

LAN
Public lan0 lan1
Pri lan2 lan3

OCR
External redundance /dev/disk/disk32

Vote
External redundance /dev/disk/disk33
Installing



Install Patch CRS 10.2.0.3(patchset ver 10.2.0.3)
./runInstaller -ignoreSysPrereqs

Node 1
crsctl stop crs
/oracle/product/crs/install/root102.sh
Node 2
crsctl stop crs
/oracle/product/crs/install/root102.sh
# crsctl will start crs automatic
After install crs patchset
vipca






Verify process
crsctl check crs
CSS appears healthy
CRS appears healthy
EVM appears healthy
ps –ef|grep d.bin
Verify config
Node 1
crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora....db1.gsd application ONLINE ONLINE mkondb1
ora....db1.ons application ONLINE ONLINE mkondb1
ora....db1.vip application ONLINE ONLINE mkondb1
ora....db2.gsd application ONLINE OFFLINE
ora....db2.ons application ONLINE OFFLINE
ora....db2.vip application OFFLINE OFFLINE
Node 2
crs_stat –t
Name Type Target State Host
------------------------------------------------------------
ora....db1.gsd application OFFLINE OFFLINE
ora....db1.ons application OFFLINE OFFLINE
ora....db1.vip application ONLINE OFFLINE
ora....db2.gsd application ONLINE ONLINE mkondb2
ora....db2.ons application ONLINE ONLINE mkondb2
ora....db2.vip application ONLINE ONLINE mkondb2
olsnodes -v
prlslms: Initializing LXL global
prlsndmain: Initializing CLSS context
prlsmemberlist: No of cluster members configured = 256
prlsmemberlist: Getting information for nodenum = 1
prlsmemberlist: node_name = mkondb1
prlsmemberlist: ctx->lsdata->node_num = 1
prls_printdata: Printing the node data
mkondb1
prlsmemberlist: Getting information for nodenum = 2
prlsmemberlist: node_name = mkondb2
prlsmemberlist: ctx->lsdata->node_num = 2
prls_printdata: Printing the node data
mkondb2
prlsndmain: olsnodes executed successfully
prlsndterm: Terminating LSF
olsnodes
mkondb1
mkondb2
ocrcheck
Status of Oracle Cluster Registry is as follows :
Version : 2
Total space (kbytes) : 2097064
Used space (kbytes) : 2016
Available space (kbytes) : 2095048
ID : 711156396
Device/File Name : /dev/rdisk/disk32
Device/File integrity check succeeded
Device/File not configured
Cluster registry integrity check succeeded
Verify cluster post
cluvfy stage -post hwos -n all
Node 1
ocrcheck
Status of Oracle Cluster Registry is as follows :
Version : 2
Total space (kbytes) : 2097064
Used space (kbytes) : 2016
Available space (kbytes) : 2095048
ID : 711156396
Device/File Name : /dev/rdisk/disk32
Device/File integrity check succeeded
Device/File not configured
Cluster registry integrity check succeeded






4. Install the Database Software and Create an ASM Instance
Install Oracle ASM (DB ver 10.2.0.1)
./runInstaller -ignoreSysPrereqs











Install Patch ASM (patchset 10.2.0.3)
./runInstaller -ignoreSysPrereqs




Install Oracle DB 10.2.0.1
./runInstaller -ignoreSysPrereqs









Install Patch Oracle DB 10.2.0.3(patchset ver 10.2.0.3)
./runInstaller -ignoreSysPrereqs








Create listener before create ASM instance
/oracle/product/db_1/bin/netca






































/oracle/product/db_1/network/admin/listener.ora
SID_LIST_LISTENER_MKONDB1 =
(SID_LIST =
(SID_DESC =
(SID_NAME = PLSExtProc)
(ORACLE_HOME = /oracle/product/db_1)
(PROGRAM = extproc)
)
)
LISTENER_MKONDB1 =
(DESCRIPTION_LIST =
(DESCRIPTION =
(ADDRESS = (PROTOCOL = TCP)(HOST = mkondb1-vip)(PORT = 1521)(IP = FIRST))
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.10.13)(PORT = 1521)(IP = FIRST))
)
)
/oracle/product/db_1/network/admin/tnsnames.ora

ASM =
(DESCRIPTION =
(ADDRESS_LIST =
(ADDRESS = (PROTOCOL = TCP)(HOST = mkondb1-vip)(PORT = 1521))
(ADDRESS = (PROTOCOL = TCP)(HOST = mkondb2-vip)(PORT = 1521))
)
(CONNECT_DATA =
(SERVICE_NAME = ASM)
)
)
EXTPROC_CONNECTION_DATA =
(DESCRIPTION =
(ADDRESS_LIST =
(ADDRESS = (PROTOCOL = IPC)(KEY = EXTPROC0))
)
(CONNECT_DATA =
(SID = PLSExtProc)
(PRESENTATION = RO)
)
)

Create ASM instance
/oracle/product/asm/bin/dbca














5. Create a Database using DBCA
Create DB instance
/oracle/product/db_1/bin/dbca

































## No need to config netca after create db
Test connect to emconsole
http://172.16.129.13:1158/em/console/aboutApplication










6. TNS and Listener configuration

Enable TAF from Net-Service After Install and not config to service
Edit /oracle/product/db_1/network/admin/tnsnames.ora
From
ONLINE =
(DESCRIPTION =
(ADDRESS = (PROTOCOL = TCP)(HOST = mkondb1-vip)(PORT = 1521))
(ADDRESS = (PROTOCOL = TCP)(HOST = mkondb2-vip)(PORT = 1521))
(LOAD_BALANCE = yes)
(CONNECT_DATA =
(SERVER = DEDICATED)
(SERVICE_NAME = ONLINE)
)
)
)
-- to --
ONLINE =
(DESCRIPTION =
(ADDRESS = (PROTOCOL = TCP)(HOST = mkondb1-vip)(PORT = 1521))
(ADDRESS = (PROTOCOL = TCP)(HOST = mkondb2-vip)(PORT = 1521))
(LOAD_BALANCE = yes)
(CONNECT_DATA =
(SERVER = DEDICATED)
(SERVICE_NAME = ONLINE)
(FAILOVER_MODE =
(TYPE = SELECT)
(METHOD = BASIC)
(RETRIES = 180)
(DELAY = 5)
)
)
)




7. Enable Archive log file and Shared NFS
Enable Archive log file
On Node1
su – oracle
export ORACLE_SID=ONLINE1
sqlplus "/ as sysdba"
ALTER SYSTEM SET log_archive_dest_1 = "location=/oracle/archive/ONLINE1" SCOPE=SPFILE SID='ONLINE1';
ON Node2
su – oracle
export ORACLE_SID=ONLINE2
sqlplus "/ as sysdba"
ALTER SYSTEM SET log_archive_dest_1 = "location=/oracle/archive/ONLINE2" SCOPE=SPFILE SID='ONLINE2';
!srvctl stop database -d ONLINE
conn / as sysdba
startup mount
archive log list
alter database archivelog;
archive log list;
alter database open;
!srvctl start instance -d ONLINE -i ONLINE2
conn / as sysdba
archive log list
ALTER SYSTEM ARCHIVE LOG CURRENT;
! ls /oracle/archive/ONLINE1
!remsh mkondb2 ls /oracle/archive/ONLINE2
# ALTER SYSTEM CHECKPOINT LOCAL affects only the instance to which you are currently connected, rather than the default instance or all instances.
#ALTER SYSTEM CHECKPOINT or ALTER SYSTEM CHECKPOINT GLOBAL affects all instances in the cluster database.
# ALTER SYSTEM SWITCH LOGFILE affects only the current instance.
# ALTER SYSTEM ARCHIVE LOG CURRENT To force a global log switch

Shared NFS
Enable NFS server 2 machine
sam-->







Node1
/etc/dfs/dfstab
share -F nfs -o rw="mkondb2" -d "archivelog dirs" /oracle/archive/ONLINE1
shareall
share –or -- exportfs
mkdir /oracle/archive/ONLINE2
/etc/fstab
mkondb2:/oracle/archive/ONLINE2 /oracle/archive/ONLINE2 nfs defaults 0 0

Node2
/etc/dfs/dfstab
share -F nfs -o rw=" mkondb1" -d "archivelog dirs" /oracle/archive/ONLINE2
shareall
share –or -- exportfs
mkdir /oracle/archive/ONLINE1
/etc/fstab
mkondb1:/oracle/archive/ONLINE1 /oracle/archive/ONLINE1 nfs defaults 0 0