Critical Assessment
Rapidly diagnose repository health and identify critical failure points
Repository Health Diagnostics
Integrity Check
git fsck --full --no-reflogs --unreachable --dangling
Complete filesystem check
git verify-pack -v .git/objects/pack/*.idx
Verify pack file integrity
git count-objects -v
Object database statistics
Corruption Detection
git cat-file -t <object-hash>
Test object readability
git show-ref --verify refs/heads/main
Verify reference integrity
find .git -name "*.idx" -exec git verify-pack {} \;
Batch pack verification
Performance Analysis
git gc --aggressive --prune=now
Aggressive cleanup test
git log --oneline --all | wc -l
Commit count analysis
du -sh .git/
Repository size check
Emergency Triage System
CRITICAL - Immediate Action Required
HIGH - Urgent Response Needed
MEDIUM - Planned Recovery
Emergency Recovery
Execute critical recovery operations to restore repository functionality
Critical Recovery Scenarios
🔥 Scenario 1: Complete Repository Corruption
Emergency Recovery Procedure:
cp -r .git .git-backup-$(date +%Y%m%d_%H%M%S)
Create emergency backup before any recovery attempts
# Extract salvageable objects
mkdir recovery-objects
git cat-file --batch-check --batch-all-objects | \
while read sha1 type size; do
if git cat-file -e $sha1 2>/dev/null; then
git cat-file $type $sha1 > recovery-objects/$sha1
fi
done
# Rebuild object database
git hash-object -w recovery-objects/*
# Find branch heads from reflogs
git reflog --all | grep "checkout: moving from" | \
awk '{print $NF}' | sort -u > potential-branches.txt
# Verify and recreate branches
while read branch; do
if git rev-parse --verify $branch >/dev/null 2>&1; then
echo "Recreating branch: $branch"
git branch -f recovered-$branch $branch
fi
done < potential-branches.txt
git clone --bare <remote-url> .git-remote-backup
rsync -av .git-remote-backup/ .git/
Use remote as source of truth for final restoration
💀 Scenario 2: Missing Critical Commits
Forensic Recovery:
# Comprehensive reflog search
git reflog --all --date=iso --format="%h %gd %gs %s"
# Find specific patterns
git log --walk-reflogs --oneline --grep="your-search-term"
# Recover from specific reflog entry
git branch recovery-branch HEAD@{2}
# Find all dangling commits
git fsck --no-reflog --unreachable | grep commit
# Examine dangling commits
git fsck --no-reflog --unreachable | grep commit | \
cut -d' ' -f3 | while read commit; do
echo "=== Commit $commit ==="
git show --stat $commit
echo ""
done
# Create branches from recovered commits
git branch recovery-$(date +%s) <commit-hash>
# Create commit from working directory changes
git add -A
git commit -m "Emergency recovery: $(date)"
# Stash recovery
git stash list --date=iso
git stash show -p stash@{0}
git stash pop stash@{0}
📦 Scenario 3: Pack File Corruption
Pack Recovery Protocol:
# Verify pack corruption
cd .git/objects/pack
for pack in pack-*.idx; do
echo "Verifying $pack"
git verify-pack -v "$pack" || echo "CORRUPTED: $pack"
done
# Extract objects from corrupted packs
git unpack-objects < pack-*.pack
# Remove corrupted packs
rm pack-*.pack pack-*.idx
# Rebuild clean packs
cd ../..
git repack -ad
git gc --aggressive
Automated Recovery Systems
#!/bin/bash
# Emergency Git Recovery Automation Script
set -euo pipefail
REPO_PATH="${1:-.}"
BACKUP_DIR="emergency-backup-$(date +%Y%m%d_%H%M%S)"
LOG_FILE="recovery-$(date +%Y%m%d_%H%M%S).log"
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
}
emergency_backup() {
log "Creating emergency backup..."
cp -r "$REPO_PATH/.git" "$BACKUP_DIR"
log "Backup created at: $BACKUP_DIR"
}
assess_damage() {
log "Assessing repository damage..."
# Check basic repository structure
if [[ ! -d "$REPO_PATH/.git" ]]; then
log "CRITICAL: No .git directory found!"
return 3
fi
# Run filesystem check
cd "$REPO_PATH"
if ! git fsck --no-reflogs 2>&1 | tee fsck-output.txt; then
corruption_level=$(grep -c "error\|corrupt\|missing" fsck-output.txt || true)
log "Found $corruption_level corruption issues"
return $((corruption_level > 10 ? 2 : 1))
fi
log "Repository appears healthy"
return 0
}
recover_objects() {
log "Attempting object recovery..."
# Create recovery workspace
mkdir -p recovery-workspace
# Extract all readable objects
git cat-file --batch-check --batch-all-objects | \
while read sha1 type size; do
if git cat-file -e "$sha1" 2>/dev/null; then
git cat-file "$type" "$sha1" > "recovery-workspace/$sha1"
else
log "Unreadable object: $sha1"
fi
done
# Rebuild object database
for obj in recovery-workspace/*; do
if [[ -f "$obj" ]]; then
git hash-object -w "$obj" || log "Failed to re-hash: $(basename "$obj")"
fi
done
}
recover_references() {
log "Recovering branch references..."
# Backup current refs
cp -r .git/refs .git/refs-backup 2>/dev/null || true
# Try to recover from reflogs
find .git/logs -name "*" -type f | while read reflog; do
ref_name=$(echo "$reflog" | sed 's|.git/logs/||')
if [[ -f "$reflog" ]]; then
last_commit=$(tail -1 "$reflog" | cut -d' ' -f2)
if git rev-parse --verify "$last_commit" >/dev/null 2>&1; then
log "Recovering reference: $ref_name -> $last_commit"
git update-ref "$ref_name" "$last_commit"
fi
fi
done
}
main() {
log "Starting emergency Git recovery for: $REPO_PATH"
# Create backup first
emergency_backup
# Assess damage level
assess_damage
damage_level=$?
case $damage_level in
0)
log "No recovery needed - repository is healthy"
;;
1)
log "Minor corruption detected - attempting repair"
git gc --aggressive
;;
2)
log "Major corruption detected - full recovery protocol"
recover_objects
recover_references
;;
3)
log "CRITICAL: Repository structure missing - manual intervention required"
exit 1
;;
esac
# Final verification
if git fsck --no-reflogs; then
log "Recovery completed successfully!"
else
log "Recovery partially successful - manual intervention may be needed"
fi
}
# Execute main function
main "$@"
Forensic Analysis
Investigate disaster causes and prevent future failures
Git Forensics Toolkit
Timeline Reconstruction
# Complete timeline analysis
git log --all --full-history --date=iso --format="%h %ad %an %s" --graph
# Find when corruption occurred
git reflog --all --date=iso | grep -E "(corrupt|error|fail)"
# Analyze specific time periods
git log --since="2024-11-01" --until="2024-11-02" --all --oneline
Object Forensics
# Analyze object relationships
git rev-list --objects --all | \
sort | uniq | \
while read sha1 name; do
size=$(git cat-file -s "$sha1" 2>/dev/null || echo "CORRUPT")
type=$(git cat-file -t "$sha1" 2>/dev/null || echo "CORRUPT")
echo "$sha1 $type $size $name"
done > object-inventory.txt
# Find largest objects (potential corruption sources)
git rev-list --objects --all | \
git cat-file --batch-check='%(objectname) %(objecttype) %(objectsize)' | \
sort -k3 -n | tail -20
Corruption Pattern Analysis
# Pattern detection script
#!/bin/bash
analyze_corruption_patterns() {
echo "=== Corruption Pattern Analysis ==="
# Check for common corruption signatures
git fsck 2>&1 | while read line; do
case "$line" in
*"corrupt"*)
echo "CORRUPTION: $line"
;;
*"missing"*)
echo "MISSING: $line"
;;
*"dangling"*)
echo "DANGLING: $line"
;;
esac
done | sort | uniq -c | sort -nr
# Analyze pack file health
echo -e "\n=== Pack File Analysis ==="
find .git/objects/pack -name "*.pack" | while read pack; do
echo "Checking: $pack"
git verify-pack -v "$pack" 2>&1 | grep -E "(corrupt|error|bad)"
done
# Repository statistics
echo -e "\n=== Repository Statistics ==="
git count-objects -v
}
Disaster Prevention Protocols
Automated Backup Systems
# Setup multiple remotes for redundancy
git remote add backup-github git@github.com:user/repo-backup.git
git remote add backup-gitlab git@gitlab.com:user/repo-backup.git
git remote add local-backup /path/to/local/backup.git
# Automated sync script
#!/bin/bash
for remote in origin backup-github backup-gitlab local-backup; do
echo "Syncing to $remote..."
git push "$remote" --all --tags || echo "Failed to sync $remote"
done
# Cron job for daily verification (add to crontab)
0 2 * * * cd /path/to/repo && git fsck --full > /tmp/git-fsck.log 2>&1
# Weekly comprehensive backup
0 0 * * 0 cd /path/to/repo && git bundle create backup-$(date +%Y%m%d).bundle --all
Monitoring & Alerting
# Repository health monitoring script
#!/bin/bash
REPO_PATH="$1"
WEBHOOK_URL="$2" # Slack/Teams webhook for alerts
check_repository_health() {
cd "$REPO_PATH"
# Run comprehensive checks
errors=0
# Filesystem check
if ! git fsck --no-reflogs >/dev/null 2>&1; then
errors=$((errors + 1))
echo "ERROR: Repository corruption detected"
fi
# Pack verification
find .git/objects/pack -name "*.pack" | while read pack; do
if ! git verify-pack "$pack" >/dev/null 2>&1; then
errors=$((errors + 1))
echo "ERROR: Pack corruption in $pack"
fi
done
# Size monitoring
repo_size=$(du -sm .git | cut -f1)
if [[ $repo_size -gt 1000 ]]; then # Alert if > 1GB
echo "WARNING: Repository size is ${repo_size}MB"
fi
# Send alerts if errors found
if [[ $errors -gt 0 ]]; then
send_alert "Repository health check failed with $errors errors"
fi
}
send_alert() {
if [[ -n "$WEBHOOK_URL" ]]; then
curl -X POST "$WEBHOOK_URL" \
-H 'Content-Type: application/json' \
-d "{\"text\":\"🚨 Git Repository Alert: $1\"}"
fi
}
check_repository_health
Team Disaster Coordination
Coordinate emergency response across development teams during critical incidents
Emergency Communication Protocols
📢 Stage 1: Immediate Alert (0-5 minutes)
Immediately notify all team members via primary communication channels
Repository: [REPO_NAME]
Impact: [BRIEF_DESCRIPTION]
Status: Investigation in progress
Lead: [YOUR_NAME]
Next Update: [TIME]
All team members stop pushing/pulling until recovery complete
git config --global push.default nothing
🔍 Stage 2: Assessment & Isolation (5-15 minutes)
# Team coordination script
#!/bin/bash
echo "=== EMERGENCY ASSESSMENT PROTOCOL ==="
echo "Repository: $(pwd)"
echo "Time: $(date)"
echo "Operator: $(git config user.name)"
echo ""
# Quick health check
echo "Repository Status:"
git status --porcelain
echo ""
echo "Recent Activity:"
git log --oneline -10
echo ""
echo "Remote Sync Status:"
git remote -v | while read remote url type; do
if [[ "$type" == "(fetch)" ]]; then
echo "Testing $remote..."
git ls-remote "$remote" >/dev/null 2>&1 && echo "✓ $remote OK" || echo "✗ $remote FAILED"
fi
done
🛠️ Stage 3: Recovery Coordination (15+ minutes)
🎯 Incident Commander
- Overall incident coordination
- Communication with stakeholders
- Decision making authority
- Recovery progress tracking
🔧 Technical Lead
- Execute recovery procedures
- Technical decision making
- Tool and script development
- Risk assessment
📋 Documentation Lead
- Incident timeline tracking
- Recovery step documentation
- Lessons learned capture
- Post-incident reporting
Team Recovery Coordination Tools
# Multi-team recovery coordination script
#!/bin/bash
INCIDENT_ID="INC-$(date +%Y%m%d%H%M)"
TEAM_REPOS="repo1 repo2 repo3" # List of affected repositories
COORDINATION_FILE="incident-${INCIDENT_ID}.log"
setup_coordination() {
echo "=== Git Disaster Recovery Coordination ===" > "$COORDINATION_FILE"
echo "Incident ID: $INCIDENT_ID" >> "$COORDINATION_FILE"
echo "Start Time: $(date)" >> "$COORDINATION_FILE"
echo "Coordinator: $(git config user.name)" >> "$COORDINATION_FILE"
echo "" >> "$COORDINATION_FILE"
}
assess_team_repositories() {
echo "Repository Assessment:" >> "$COORDINATION_FILE"
for repo in $TEAM_REPOS; do
echo "--- Checking $repo ---" >> "$COORDINATION_FILE"
if [[ -d "$repo/.git" ]]; then
cd "$repo"
# Health check
if git fsck --no-reflogs >/dev/null 2>&1; then
echo "✓ $repo: Healthy" >> "../$COORDINATION_FILE"
else
error_count=$(git fsck 2>&1 | grep -c "error\|corrupt" || echo "0")
echo "✗ $repo: $error_count errors detected" >> "../$COORDINATION_FILE"
fi
# Remote status
git remote | while read remote; do
if git ls-remote "$remote" >/dev/null 2>&1; then
echo " ✓ Remote $remote: Accessible" >> "../$COORDINATION_FILE"
else
echo " ✗ Remote $remote: Unreachable" >> "../$COORDINATION_FILE"
fi
done
cd ..
else
echo "✗ $repo: Repository not found" >> "$COORDINATION_FILE"
fi
echo "" >> "$COORDINATION_FILE"
done
}
coordinate_team_backup() {
echo "Creating team-wide emergency backups..." >> "$COORDINATION_FILE"
mkdir -p "emergency-backups-$INCIDENT_ID"
for repo in $TEAM_REPOS; do
if [[ -d "$repo/.git" ]]; then
echo "Backing up $repo..." >> "$COORDINATION_FILE"
tar -czf "emergency-backups-$INCIDENT_ID/${repo}-backup.tar.gz" "$repo"
fi
done
}
generate_recovery_plan() {
cat >> "$COORDINATION_FILE" << 'EOF'
=== RECOVERY PLAN ===
Phase 1: Immediate Stabilization
□ Stop all team development work
□ Create emergency backups of all repositories
□ Assess damage extent across all repos
□ Establish communication channels
Phase 2: Repository Recovery
□ Identify least damaged repository as reference
□ Execute recovery procedures on critical repos
□ Verify recovered repositories
□ Test remote connectivity
Phase 3: Team Coordination
□ Coordinate team workspace updates
□ Verify all team members can access repos
□ Resume development with caution
□ Monitor for recurring issues
Phase 4: Post-Incident
□ Document lessons learned
□ Implement prevention measures
□ Update team disaster protocols
□ Schedule team retrospective
EOF
}
main() {
setup_coordination
assess_team_repositories
coordinate_team_backup
generate_recovery_plan
echo "Coordination setup complete. See: $COORDINATION_FILE"
echo "Share this file with all team members."
}
main "$@"
Emergency Response Mastery
Critical disaster recovery skills acquired - Ready for any Git emergency
Emergency Response Capabilities
Diagnostic Mastery
Recovery Expertise
Forensic Analysis
Team Coordination
Git Emergency Response Specialist
Certification Date:
Emergency Skills Mastered: Critical Assessment, Emergency Recovery, Forensic Analysis, Team Coordination
🚨 Emergency Response Ready!
You now possess the critical skills to handle any Git disaster. From repository corruption to team-wide emergencies, you can assess, recover, and coordinate under pressure.