620.20 a

#600#ML_Libraries_and_Implementation#620#Data_visualization#620.20#Script#620.20 a#Shell_Scirpt_For_visualize_GPU_usage

Record Gpu max and taken time script

#Linux

#!/bin/bash
max=0
subject=$1
log_file="${subject}_$(date '+%Y-%m-%d').log" # Log file for the whole day
process_pattern="scripts/sampling/" # Unique pattern in your process command
start_time=0
found_process=0

while true; do
    # Get individual GPU usages
    individual_usages=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '{print $1" MiB"}' | paste -sd ", ")
    # Sum of all GPU usages
    usage=$(echo $individual_usages | tr -d ' MiB' | awk '{split($0,a,", "); sum=0; for(i in a) sum+=a[i]; print sum}')
    # Update the max usage if the current sum is greater
    if [ "$usage" -gt "$max" ]; then
        max=$usage
    fi

    # Define your message
    current_time=$(date '+%Y-%m-%d-%H-%M-%S') # Update current time in each loop
    message="Current usage: $individual_usages, Max usage: $max MiB, subject: ${subject}, Time: $current_time"

    # Display the message
    echo "$message"

    # Append the message to the day's log file
    echo "$message" >> "$log_file"

    ######## Record Taken time######
    # Check if a process matching the pattern is running
    if pgrep -f "$process_pattern" > /dev/null; then
        if [ $start_time -eq 0 ]; then
            start_time=$(date +%s) # Record start time when process is first found
            found_process=1
        fi
    else
        if [ $found_process -eq 1 ]; then
            end_time=$(date +%s) # Record end time when process no longer found
            elapsed=$((end_time - start_time))
            # Record the final log with total time and max GPU usage
            echo "####################################################" >> "$log_file"
            echo "Time taken by process with pattern '$process_pattern': $elapsed seconds, Max usage: $max MiB" >> "$log_file"
            break # Exit the loop since the process has finished
        fi
    fi
    sleep 5 # Adjust the sleep time as