Oracle DR: Fix database verification, add bash log, and collect full RMAN log

Critical fixes and improvements:

1. Database verification fix (robust):
   - Use Select-String -Quiet to get True/False boolean
   - Convert PowerShell boolean to bash-friendly format
   - Check for 'READ WRITE' in entire sqlplus output
   - Eliminates false negatives from text parsing issues

2. Collect FULL RMAN restore log:
   - Removed -Head 200 limitation
   - Now sends complete RMAN log in email
   - Better debugging with full context
   - Updated templates: "first 200 lines" → "complete"

3. Add bash script log to email notifications:
   - Include last 100 lines of bash execution log
   - Separate "RMAN Restore Log" and "Bash Script Log" sections
   - Both text and HTML templates updated
   - Shows script flow and any bash-level errors

This fixes the issue where 42,625 tables were restored successfully
but test reported FAILED due to query output format mismatch.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>

oracle/standby-server-scripts/weekly-dr-test-proxmox.sh

@@ -90,12 +90,17 @@ WARNINGS
 {{/each}}
 {{/if}}
 
-RESTORE LOG (first 200 lines)
+RMAN RESTORE LOG (complete)
 ---
 {{restore_log}}
 ---
 
-Log: {{log_file}}
+BASH SCRIPT LOG (last 100 lines)
+---
+{{bash_log}}
+---
+
+Full log: {{log_file}}
 Next test: Saturday 06:00
 EOF
 
@@ -183,15 +188,24 @@ EOF
     <tr>
       <td style="padding:16px 0 0 0;">
         <table style="width:100%;border-collapse:collapse;font-size:12px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
-          <tr><td style="padding:8px 12px;font-weight:600;font-size:13px;">Restore Log (first 200 lines)</td></tr>
+          <tr><td style="padding:8px 12px;font-weight:600;font-size:13px;">RMAN Restore Log (complete)</td></tr>
           <tr><td style="padding:8px 12px;font-family:monospace;white-space:pre-wrap;word-wrap:break-word;border-top:1px solid #e1e4e8;">{{restore_log}}</td></tr>
         </table>
       </td>
     </tr>
 
+    <tr>
+      <td style="padding:16px 0 0 0;">
+        <table style="width:100%;border-collapse:collapse;font-size:12px;border:1px solid #e1e4e8;border-radius:4px;background:#f9fafb;">
+          <tr><td style="padding:8px 12px;font-weight:600;font-size:13px;">Bash Script Log (last 100 lines)</td></tr>
+          <tr><td style="padding:8px 12px;font-family:monospace;white-space:pre-wrap;word-wrap:break-word;border-top:1px solid #e1e4e8;">{{bash_log}}</td></tr>
+        </table>
+      </td>
+    </tr>
+
     <tr>
       <td style="padding:16px 0 0 0;font-size:12px;color:#6c757d;">
-        Log: {{log_file}} · Next test: Saturday 06:00
+        Full log: {{log_file}} · Next test: Saturday 06:00
       </td>
     </tr>
   </table>
@@ -373,9 +387,16 @@ run_dr_test() {
                 step_start=$(date +%s)
                 log "STEP 5: Verifying database"
 
-                # Use PowerShell to query database status
+                # Use PowerShell to query database status (check if contains READ WRITE anywhere)
                 db_status=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
-                    "powershell -Command \"echo 'SELECT OPEN_MODE FROM V\`\$DATABASE;' | sqlplus -s / as sysdba | Select-String 'READ WRITE'\"" 2>/dev/null || echo "")
+                    "powershell -Command \"echo 'SELECT OPEN_MODE FROM V\`\$DATABASE;' | sqlplus -s / as sysdba | Out-String | Select-String -Pattern 'READ WRITE' -Quiet\"" 2>/dev/null || echo "false")
+                
+                # Convert PowerShell True/False to bash-friendly value
+                if [[ "$db_status" == *"True"* ]] || [[ "$db_status" == "True" ]]; then
+                    db_status="READ WRITE"
+                else
+                    db_status=""
+                fi
 
                 # Use PowerShell to count tables
                 tables_restored=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
@@ -392,15 +413,15 @@ run_dr_test() {
                     track_step "Database Verification" false "Database not OPEN" "$step_start"
                 fi
 
-                # Collect restore log from VM (always attempt collection)
+                # Collect restore log from VM (always attempt collection - FULL log)
                 log "Collecting restore log from DR VM..."
                 restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
-                    "powershell -Command \"Get-Content 'D:\\oracle\\logs\\restore_from_zero.log' -Head 200 -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
+                    "powershell -Command \"Get-Content 'D:\\oracle\\logs\\restore_from_zero.log' -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
                 
                 # If not found, try alternate locations
                 if [ -z "$restore_log" ]; then
                     restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
-                        "powershell -Command \"Get-Content 'D:\\oracle\\temp\\restore_from_zero.log' -Head 200 -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
+                        "powershell -Command \"Get-Content 'D:\\oracle\\temp\\restore_from_zero.log' -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
                 fi
                 
                 # Still not found, use fallback message
@@ -419,14 +440,14 @@ run_dr_test() {
                 track_step "Cleanup" true "Database cleaned, ~${cleanup_freed}GB freed" "$step_start"
 
             else
-                # Collect restore log even when restore fails
+                # Collect restore log even when restore fails (FULL log)
                 log "Collecting restore log after failure..."
                 restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
-                    "powershell -Command \"Get-Content 'D:\\oracle\\logs\\restore_from_zero.log' -Head 200 -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
+                    "powershell -Command \"Get-Content 'D:\\oracle\\logs\\restore_from_zero.log' -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
 
                 if [ -z "$restore_log" ]; then
                     restore_log=$(ssh -p "$DR_VM_PORT" "$DR_VM_USER@$DR_VM_IP" \
-                        "powershell -Command \"Get-Content 'D:\\oracle\\temp\\restore_from_zero.log' -Head 200 -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
+                        "powershell -Command \"Get-Content 'D:\\oracle\\temp\\restore_from_zero.log' -ErrorAction SilentlyContinue\"" 2>/dev/null || echo "")
                 fi
                 
                 # Always try to get some error output from RMAN script
@@ -500,6 +521,12 @@ run_dr_test() {
     # Escape restore log for JSON
     local restore_log_json
     restore_log_json=$(echo "$restore_log" | jq -Rs .)
+    
+    # Collect last 100 lines of bash script log
+    local bash_log
+    bash_log=$(tail -100 "$LOG_FILE" 2>/dev/null || echo "Bash log not available")
+    local bash_log_json
+    bash_log_json=$(echo "$bash_log" | jq -Rs .)
 
     local json_data=$(cat <<JSON
 {
@@ -523,7 +550,8 @@ run_dr_test() {
     "vm_status": "$vm_status_label",
     "nfs_status": "${nfs_status:-Unknown}",
     "log_file": "$LOG_FILE",
-    "restore_log": $restore_log_json
+    "restore_log": $restore_log_json,
+    "bash_log": $bash_log_json
 }
 JSON
 )