On-Prem Troubleshooting Guide

opensearch            : (no dependencies)
opensearch-dashboards : opensearch
mongodb               : (no dependencies)
kafka                 : (no dependencies)
td-agent              : opensearch, kafka
scheduler             : opensearch, mongodb, kafka
analytics-manager     : mongodb, kafka
ingestion             : kafka
webui                 : api
api                   : opensearch, mongodb, kafka, cbom, sm
cbom                  : opensearch, mongodb, kafka, sm
sm                    : (no dependencies)
haproxy               : (no dependencies)

logger.deprecation.level: warn
appender.rolling.type: RollingFile
appender.rolling.policies.size.size: 100MB

<installation_path>/logs/*.log {
    daily
    rotate 7
    compress
    delaycompress
    missingok
    notifempty
    copytruncate
    create 0640 <your_user> <your_group>
}

cd $installation_path
./scripts/manage.sh <action> [options] [service1 service2 ...]

# Check all services status
./scripts/manage.sh status

# Check specific service status
./scripts/manage.sh status opensearch
./scripts/manage.sh status mongodb kafka

# Start all services
./scripts/manage.sh start

# Start specific service with debug output
./scripts/manage.sh start -d opensearch

# Stop specific services
./scripts/manage.sh stop haproxy td-agent

# Restart a service
./scripts/manage.sh restart scheduler

# Reload HAProxy configuration
./scripts/manage.sh reload haproxy

2025-01-15 10:30:45 [INFO] Service Status
opensearch             Running (PID: 12345)
mongodb                Running (PID: 12346)
kafka                  Running (PID: 12347)
webui                  Not running
api                    Running (PID: 12349)

# Check systemd service status
sudo systemctl status kf_analytics.service

# Enable automatic startup
sudo systemctl enable kf_analytics.service

# View service logs
sudo journalctl -u kf_analytics.service -f

# Check current limits
ulimit -l

# Run tune.sh command to adjust system settings
cd <installer_directory>
sudo ./scripts/tune.sh

# Important: Logout/Login to refresh session

# Check current heap settings in service environment file
cat $installation_path/services/opensearch/config/jvm.options

# Modify -Xms and -Xmx values
# Example: 
#  -Xms32g 
#  -Xmx32g

# Set certificate variables
# Important: 
#   - Replace </path/to/installation> with actual path
#   - Replace "kf-agilesec.internal" with your actual domain name

export installation_path=</path/to/installation>
export analytics_internal_domain="kf-agilesec.internal" # default: kf-agilesec.internal. Change it to match your actual domain
export CA_CERT=$installation_path/certificates/ca/agilesec-rootca-cert.pem
export CLIENT_CERT=$installation_path/certificates/$analytics_internal_domain/admin-user-cert.pem
export CLIENT_KEY=$installation_path/certificates/$analytics_internal_domain/admin-user-key.pem

# Check cluster health
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  'https://127.0.0.1:9200/_cluster/health?pretty'

# Check shard allocation explanation
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  'https://127.0.0.1:9200/_cluster/allocation/explain?pretty'

# List unassigned shards
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  'https://127.0.0.1:9200/_cat/shards?v&h=index,shard,prirep,state,unassigned.reason'

# Enable shard allocation (if disabled)
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  -X PUT 'https://127.0.0.1:9200/_cluster/settings' \
  -H 'Content-Type: application/json' -d '{
    "transient": {
      "cluster.routing.allocation.enable": "all"
    }
  }'

# Reroute stuck shards
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  -X POST 'https://127.0.0.1:9200/_cluster/reroute?retry_failed=true'

# Important: 
#   - Replace <client-cert-path> with actual path
#   - Replace <ca-cert-path> with actual path
export CLIENT_CERT=<client-cert-path>
export CA_CERT=<ca-cert-path>
# Check certificate expiration
openssl x509 -in $CLIENT_CERT -noout -dates

# Verify certificate chain
openssl verify -CAfile $CA_CERT $CLIENT_CERT

# Check certificate subject/issuer
openssl x509 -in $CLIENT_CERT -noout -subject -issuer

# Check current heap usage
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  'https://127.0.0.1:9200/_nodes/stats/jvm?pretty' | grep -A 10 "heap"

# Check current heap settings in service environment file
cat $installation_path/services/opensearch/config/jvm.options

# Modify -Xms and -Xmx values
# Example: 
#  -Xms32g 
#  -Xmx32g

# Restart OpenSearch
./scripts/manage.sh restart opensearch

# Check disk usage
# Important: 
#   - Replace </path/to/installation> with actual path
#   - Replace "kf-agilesec.internal" with your actual domain name

export installation_path=</path/to/installation>
export analytics_internal_domain="kf-agilesec.internal" # default: kf-agilesec.internal. Change it to match your actual domain
export CA_CERT=$installation_path/certificates/ca/agilesec-rootca-cert.pem
export CLIENT_CERT=$installation_path/certificates/$analytics_internal_domain/admin-user-cert.pem
export CLIENT_KEY=$installation_path/certificates/$analytics_internal_domain/admin-user-key.pem

# Check disk size of each nodes
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  'https://127.0.0.1:9200/_cat/allocation?v&h=node,shards,disk.percent,disk.used,disk.avail,disk.total'

# Remove stale lock file (only if mongod is not running)
rm -f $installation_path/data/mongodb/mongod.lock

# Test MongoDB connection with TLS
# Important: 
#   - Replace </path/to/installation> with actual path
#   - Replace "kf-agilesec.internal" with your actual domain name
#   - Replace "<Path to installer-directory>" with actual path

export installation_path=</path/to/installation>
export analytics_internal_domain="kf-agilesec.internal" # default: kf-agilesec.internal. Change it to match your actual domain
export installer_path=<Path to installer-directory> #Note that <installer_path> is different from <installation_path>
export OPENSSL_CONF="$installer_path/templates/mongodb/openssl-mongosh.cnf"


$installation_path/bin/mongosh \
  --tls \
  --tlsCAFile $installation_path/certificates/ca/agilesec-rootca-cert.pem \
  --tlsCertificateKeyFile $installation_path/certificates/$analytics_internal_domain/shared-client-combo-cert-key.pem \
  --host 127.0.0.1 \
  --authenticationMechanism MONGODB-X509 \
  --port 27017 \
  --eval 'db.runCommand({ ping: 1 })'

# Check if ports are in use
ss -tlnp | grep -E '(9092|9093|9094)'

# Check Kafka logs
tail -100 $installation_path/services/kafka_*/logs/server.log

# Find and kill process using the port
fuser -k 9092/tcp
fuser -k 9093/tcp
fuser -k 9094/tcp

# Restart Kafka
./scripts/manage.sh start kafka

# Verify keystore exists and is readable
export analytics_internal_domain="kf-agilesec.internal" # default: kf-agilesec.internal. Change it to match your actual domain
export sm_keystore_pass=$(cat $installation_path/certificates/$analytics_internal_domain/sm-service-keystore.pass)

ls -la $installation_path/certificates/$analytics_internal_domain/sm-service.p12

# Verify password
$installation_path/bin/java/bin/keytool -list -keystore $installation_path/certificates/$analytics_internal_domain/sm-service.p12 -storepass "$sm_keystore_pass"

# Set up certificate environment
export installation_path=</path/to/installation>
export analytics_internal_domain="kf-agilesec.internal" # default: kf-agilesec.internal. Change it to match your actual domain
export CA_CERT=$installation_path/certificates/ca/agilesec-rootca-cert.pem
export CLIENT_CERT=$installation_path/certificates/$analytics_internal_domain/admin-user-cert.pem
export CLIENT_KEY=$installation_path/certificates/$analytics_internal_domain/admin-user-key.pem

# Important: 
#   - Replace </path/to/installation> with actual path
#   - Replace "kf-agilesec.internal" with your actual domain name
# Cluster health
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  'https://127.0.0.1:9200/_cluster/health?pretty'

# Node status
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  'https://127.0.0.1:9200/_cat/nodes?v&h=ip,name,role,master,heap.percent,disk.used_percent'

# Cluster statistics
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  'https://127.0.0.1:9200/_cluster/stats?pretty'

# Index health
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  'https://127.0.0.1:9200/_cat/indices?v'

# Important: 
#   - Replace </path/to/installation> with actual path
#   - Replace "kf-agilesec.internal" with your actual domain name
#   - Replace "<Path to installer-directory>" with actual path
export installation_path=</path/to/installation>
export analytics_internal_domain="kf-agilesec.internal" # default: kf-agilesec.internal. Change it to match your actual domain
export installer_path=<Path to installer-directory> #Note that <installer_path> is different from <installation_path>
export OPENSSL_CONF="$installer_path/templates/mongodb/openssl-mongosh.cnf"

# Connect and check replica set status
$installation_path/bin/mongosh \
  --tls \
  --tlsCAFile $installation_path/certificates/ca/agilesec-rootca-cert.pem \
  --tlsCertificateKeyFile $installation_path/certificates/$analytics_internal_domain/admin-user-combo-cert-key.pem \
  --authenticationMechanism MONGODB-X509 \
  --host 127.0.0.1 \
  --port 27017 \
  --eval 'rs.status()'

# Check replica set members
$installation_path/bin/mongosh \
  --tls \
  --tlsCAFile $installation_path/certificates/ca/agilesec-rootca-cert.pem \
  --tlsCertificateKeyFile $installation_path/certificates/$analytics_internal_domain/admin-user-combo-cert-key.pem \
  --authenticationMechanism MONGODB-X509 \
  --host 127.0.0.1 \
  --port 27017 \
  --quiet \
  --eval 'const h=db.hello(); print("Members:", h.hosts.length); printjson(h.hosts)'

export installation_path=</path/to/installation>
export analytics_internal_domain="kf-agilesec.internal" # default: kf-agilesec.internal. Change it to match your actual domain

CA_PEM="$installation_path/certificates/ca/agilesec-rootca-cert.pem"
CLIENT_CERT="$installation_path/certificates/$analytics_internal_domain/shared-client-cert.pem"
CLIENT_KEY="$installation_path/certificates/$analytics_internal_domain/shared-client-key.pem"
CLIENT_COMBO="/tmp/client-combo-key-cert.pem"

# 2. Create a combined PEM file (Required for ssl.keystore.location in Java)
# The order usually doesn't matter, but Key + Cert is standard.
cat "$CLIENT_KEY" "$CLIENT_CERT" > "$CLIENT_COMBO"

# 3. Generate the ssl.properties file
cat > /tmp/ssl.properties << EOF
security.protocol=SSL

# Truststore (The CA Certificate)
ssl.truststore.type=PEM
ssl.truststore.location=$CA_PEM

# Keystore (The Client Key + Certificate)
ssl.keystore.type=PEM
ssl.keystore.location=$CLIENT_COMBO

# If your Private Key is encrypted, uncomment the line below:
# ssl.key.password=$(cat $installation_path/certificates/$analytics_internal_domain/agilesec-client-keystore.pass)
EOF

# Important: 
#   - Replace </path/to/installation> with actual path
#   - Replace "kf-agilesec.internal" with your actual domain name

# List all topics
export JAVA_HOME=$installation_path/bin/java
$installation_path/services/kafka_*/bin/kafka-topics.sh \
  --bootstrap-server 127.0.0.1:9092 \
  --command-config /tmp/ssl.properties \
  --list

# Describe all consumer groups
$installation_path/services/kafka_*/bin/kafka-consumer-groups.sh \
  --bootstrap-server 127.0.0.1:9092 \
  --command-config /tmp/ssl.properties \
  --describe --all-groups

# Check broker metadata
$installation_path/services/kafka_*/bin/kafka-dump-log.sh \
  --files $installation_path/data/kafka/__cluster_metadata-0/00000000000000000000.log \
  --cluster-metadata-decoder

# 1. Stop all services gracefully
./scripts/manage.sh stop

# 2. Verify all processes are stopped
./scripts/manage.sh status
ps aux | grep -E '(opensearch|mongod|kafka|java|node)'

# 3. Verify disk space
df -h

# 4. Start services in order
./scripts/manage.sh start 

# 5. Verify all services
./scripts/manage.sh status

# On the failed node:

# 1. Stop OpenSearch
./scripts/manage.sh stop opensearch

# 2. Check and fix data directory if corrupted. Backup the directory before deleting.
# (Only if necessary, e.g., when recovering from backup - THIS WILL RESULT IN DATA LOSS)
# rm -rf $installation_path/data/opensearch/node

# 3. Restart OpenSearch
./scripts/manage.sh start opensearch

# 4. Monitor cluster recovery
watch -n 5 "curl -s -k --cacert \$CA_CERT --cert \$CLIENT_CERT --key \$CLIENT_KEY \
  'https://127.0.0.1:9200/_cluster/health?pretty' | grep -E '(status|relocating|initializing)'"

# 1. Stop MongoDB on failed secondary
./scripts/manage.sh stop mongodb

# 2. Optionally resync from primary (if data is corrupted)
# rm -rf $installation_path/data/mongodb/*

# 3. Start MongoDB
./scripts/manage.sh start mongodb

# 4. MongoDB will automatically sync from primary

# If primary fails, a secondary should automatically become primary
# After recovering the failed node:

# 1. Start MongoDB
./scripts/manage.sh start mongodb

# 2. It will rejoin as secondary and sync
# 3. To force it back to primary (if desired):
# rs.stepDown()  # On current primary

# 1. Stop failed broker
./scripts/manage.sh stop kafka

# 2. Check for log corruption
ls -la $installation_path/data/kafka/

# 3. Start broker
./scripts/manage.sh start kafka

# 4. Verify broker rejoined cluster
$installation_path/services/kafka_*/bin/kafka-dump-log.sh \
  --files $installation_path/data/kafka/__cluster_metadata-0/00000000000000000000.log \
  --cluster-metadata-decoder

# Phase 1: Infrastructure (run on all nodes)
./scripts/manage.sh start opensearch mongodb kafka

# Phase 2: Wait for infrastructure to stabilize
sleep 120

# Verify infrastructure health on each node
./scripts/manage.sh status opensearch mongodb kafka

# Phase 3: Supporting services
./scripts/manage.sh start haproxy td-agent

# Phase 4: Application services
./scripts/manage.sh start sm analytics-manager ingestion scheduler
./scripts/manage.sh start webui api cbom

# Phase 5: Final verification
./scripts/manage.sh status

# CPU usage by process
top -b -n 1 | head -20

# Memory usage
free -h
ps aux --sort=-%mem | head -10

# Enable slow query log
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  -X PUT 'https://127.0.0.1:9200/_all/_settings' \
  -H 'Content-Type: application/json' -d '{
    "index.search.slowlog.threshold.query.warn": "10s",
    "index.search.slowlog.threshold.query.info": "5s"
  }'

# Check hot threads
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  'https://127.0.0.1:9200/_nodes/hot_threads'

# Check pending tasks
curl -k --cacert $CA_CERT --cert $CLIENT_CERT --key $CLIENT_KEY \
  'https://127.0.0.1:9200/_cluster/pending_tasks?pretty'

// Go to mongosh
$installation_path/bin/mongosh \
  --tls \
  --tlsCAFile $installation_path/certificates/ca/agilesec-rootca-cert.pem \
  --tlsCertificateKeyFile $installation_path/certificates/$analytics_internal_domain/admin-user-combo-cert-key.pem \
  --authenticationMechanism MONGODB-X509 \
  --host 127.0.0.1 \
  --port 27017

// Enable profiling for slow queries
db.setProfilingLevel(1, { slowms: 100 })

// Check slow queries
db.system.profile.find().sort({ ts: -1 }).limit(10).pretty()

// Check current operations
db.currentOp({ "secs_running": { "$gt": 5 } })

// Server status
db.serverStatus()

# Check consumer lag
$installation_path/services/kafka_*/bin/kafka-consumer-groups.sh \
  --bootstrap-server 127.0.0.1:9092 \
  --command-config /tmp/ssl.properties \
  --describe --all-groups

# Check log segment sizes
du -sh $installation_path/data/kafka/*

#!/bin/bash
# Create support bundle
export installation_path="/home/ec2-user/Installer_package/singlenode-26122025-rhel8_340-rc1" # Update this path to your actual installation directory

BUNDLE_DIR="/tmp/support-bundle-$(date +%Y%m%d-%H%M%S)" # 
mkdir -p "$BUNDLE_DIR"

# System information
echo "=== System Info ===" > "$BUNDLE_DIR/system_info.txt"
uname -a >> "$BUNDLE_DIR/system_info.txt"
cat /etc/os-release >> "$BUNDLE_DIR/system_info.txt"
free -h >> "$BUNDLE_DIR/system_info.txt"
df -h >> "$BUNDLE_DIR/system_info.txt"
uptime >> "$BUNDLE_DIR/system_info.txt"

# Service status
$installation_path/scripts/manage.sh status > "$BUNDLE_DIR/service_status.txt" 2>&1

# Application logs (last 5000 lines each)
mkdir -p "$BUNDLE_DIR/logs"
for log in $installation_path/logs/*.log; do
  tail -5000 "$log" > "$BUNDLE_DIR/logs/$(basename $log)" 2>/dev/null
done

# Configuration (sanitized)
mkdir -p "$BUNDLE_DIR/config"
# Copy configs but remove sensitive data
for conf in $installation_path/config_envs/*; do
  grep -v -E '(PASSWORD|SECRET|KEY|TOKEN)' "$conf" > "$BUNDLE_DIR/config/$(basename $conf)" 2>/dev/null
done

# Create archive
tar -czf "${BUNDLE_DIR}.tar.gz" -C /tmp "$(basename $BUNDLE_DIR)"
rm -rf "$BUNDLE_DIR"

echo "Support bundle created: ${BUNDLE_DIR}.tar.gz"