0

I am still very new to programming in Bash but wanted to make my life easier when it comes to file modification of wordlists. I understand that some of the commands my not be working, but my main problem is getting the options to work correctly. the basic syntax of the program should be "./filemod [option] [argument] [argument]". I was able to get the '-h' option to work and display the different options of the program. I can not get the other options to work correctly, I get no output at all and no error messages to go off of. I am failing to understand what is wrong here and would appreciate any help at all.

Thank you in advance

#!/bin/bash

wordlist=$2
outbound=$3

options=alauwcntbjeeEEelpscrcrpRxmh

usage() {
  echo "Script Syntax: [ bash filemod.sh [-options] wordlist outbound file]"
  echo "Options:"
  echo "  -h       Show options"
  echo "  -al      Convert alpha characters on each line in file to lowercase characters"
  echo "  -au      Convert alpha characters on each line in file to uppercase characters"
  echo "  -wc      Create custom wordlist based off the website and print all characters that are 5 or greater"
  echo "  -nt      Clean dictionary/wordlist of newlines and tabs"
  echo "  -bj      Clean Dictionary/wordlist of binary data junk/characters left in file"
  echo "  -e       Extract all lowercase strings from each line and output to wordlist"
  echo "  -E       Extract all uppercase strings from each line and output to wordlist"
  echo "  -Ee      Extract all uppercase and lowercase strings from each line and output to wordlist"
  echo "  -el      Extract strings of a specific length into a new file/wordlist"
  echo "  -p       Pull 100 random samples from wordlist/password for visual analysis"
  echo "  -sc      Print statistics on length of each string and total counts per length"
  echo "  -rc      Remove all Duplicate strings and count how many times they are present; then sort by their count by descending order"
  echo "  -rp      Remove lines that match from both files and only print lines that have not matched from file2"
  echo "  -R       Reverse each line in the file from longest to shortest"
  echo "  -x       Split file into separate files by X number of lines per outfile"
  echo "  -m       Take your 2 files merge and remove duplicate lines and maintain order"
}

while getopts $options opt; do
  case $opt in
    h)
      usage
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      usage
      exit 1
      ;;
  esac
done

extract_lowercase() {
    # Extract all lowercase strings from each line and output to wordlist
    sed 's/[^a-z]*//g' "$wordlist" > "$outbound"
}
while getopts $options opt; do
  case $opt in
    e)
      extract_lowercase
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      extract_lowercase
      exit 1
      ;;
  esac
done

extract_uppercase() {
    # Extract all Uppercae Extract all lowercase and uppercase strings from each line and output to wordlist
    sed 's/[^A-Z]*//g' "$wordlist" > "$outbound" 
}
while getopts $options opt; do
  case $opt in
    E)
      extract_uppercase
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      extract_uppercase
      exit 1
      ;;
  esac
done
extract_lower_upper() {
    # Extract all lowercase and uppercase strings from each line and output to wordlist
    sed 's/[^a-Z]*//g' "$wordlist" > "$outbound" 
}
while getopts $options opt; do
  case $opt in
    Ee)
      extract_lower_upper
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      extract_lower_upper
      exit 1
      ;;
  esac
done
100_random() { 
    # Pull 100 random samples from wordlist/password for visual analysis
    shuf -n 100 "$wordlist"
}
while getopts $options opt; do
  case $opt in
    p)
      100_random
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      100_random
      exit 1
      ;;
  esac
done
print_stat_length() {
    # Print statistics on length of each string and total counts per length
    awk '{print length}' "$wordlist" | sort -n | uniq -c
}
while getopts $options opt; do
  case $opt in
    sc)
      print_stat_length
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      print_stat_length
      exit 1
      ;;
  esac
done
remove_dup_sort_desend() {
    # Remove all Duplicate strings and count how many times they are present; then sort by their count by decending order
    sort -nr | uniq -c "$wordlist" | sort -nr
}
while getopts $options opt; do
  case $opt in
    rc)
      remove_dup_sort_desend
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      remove_dup_sort_desend
      exit 1
      ;;
  esac
done
website_scrape() {
    # Create custom wordlist based off the website and print all characters that are 5 or greater
    curl -s -k https://"$wordlist" | grep -oE '[a-z]+' | tr '[:upper:]' '[:lower:]' | sort | uniq | sort -n | grep -E '^.{5,}$' > "$outbound"
}
while getopts $options opt; do
  case $opt in
    wc)
      website_scrape
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      website_scrape
      exit 1
      ;;
  esac
done
remove_and_replace() {
    # remove lines that match from both files and only print lines that have not matched from file2
    grep -vwf -f "$wordlist" "$outbound"
}
while getopts $options opt; do
  case $opt in
    rp)
      remove_and_replace
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      remove_and_replace
      exit 1
      ;;
  esac
done
merge_and_remove() {
    # Take your 2 files merge and remove duplicate lines and maintain order
    file1=$wordlist
    file2=$outbound
    outbound="merged_no_duplicates.txt"
    # Remove the output file if it already exists
    rm -f "$outbound"
    # Concatenate the contents of file1 and file2 into the outbound file
    cat "$file1" "$file2" >> "$outbound"
    # Sort the outbound file and remove duplicates
    sort "$outbound" | uniq > "$outbound.tmp"
    # Replace the original outbound file with the deduplicated file
    mv "$outbound.tmp" "$outbound"
}
while getopts $options opt; do
  case $opt in
    m)
      merge_and_remove
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      merge_and_remove
      exit 1
      ;;
  esac
done
extract_string_length() {
    # Extract strings of a specific length into a new file/wordlist
    awk 'length == 8' "$wordlist" > "$outbound"
}
while getopts $options opt; do
  case $opt in
    el)
      extract_string_length
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      extract_string_length
      exit 1
      ;;
  esac
done
convert_lowercase() {
    # Convert alpha characters on each line in file to lowercase characters
    tr "A-Z" "a-z" < "$wordlist" > "$outbound"  
}
while getopts $options opt; do
  case $opt in
    al)
      convert_lowercase
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      convert_lowercase
      exit 1
      ;;
  esac
done
convert_uppercase() {
    # Convert alpha characters on each line in file to uppercase characters
    tr "a-z" "A-Z" < "$wordlist" > "$outbound"  
}
while getopts $options opt; do
  case $opt in
    au)
      convert_uppercase
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      convert_uppercase
      exit 1
      ;;
  esac
done
split_file() {
    # Split file into separate files by X number of lines per outfile
    split -d -l 3000 "$wordlist" "$outbound"
}
while getopts $options opt; do
  case $opt in
    x)
      split_file
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      split_file
      exit 1
      ;;
  esac
done
revsere_longto_short() {
    # Reverse each line in the file from longest to shortest
    awk '{print length,$0} " " $0; }' "$wordlist" | sort -r -n | cut -d ' ' -f2-
}
while getopts $options opt; do
  case $opt in
    R)
      revsere_longto_short
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      revsere_longto_short
      exit 1
      ;;
  esac
done
bye_newline_tab() {
    # Clean dictionary/wordlist of newlines and tabs
    cat "$wordlist" | tr -cd "[:print:][/n/t]\n" > "$outbound"
}
while getopts $options opt; do
  case $opt in
    nt)
      bye_newline_tab
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      bye_newline_tab
      exit 1
      ;;
  esac
done
bye_binary_junk() {
    # Clean Dictionary/wordlist of binary data junk/characters left in file
    tr -cd '\11\12\15\40-\176' < "$wordlist" > "$outbound"
}
while getopts $options opt; do
  case $opt in
    bj)
      bye_binary_junk
      exit 0
      ;;
    \?)
      echo "Invalid option: -$opt" >&2
      bye_binary_junk
      exit 1
      ;;
  esac
done

This was another version of the code I tried and could not get it to work. This is not the whole code just a sample ow what was tried. I was mostly testing the '-e' option against a simple wordlist and there is 0 output I can not figure out how to reference the correct option for the syntax in terminal.

execute_action(){
    case $1 in
        "-e")
            # Extract all lowercase strings from each line and output to wordlist
            sed 's/[^a-z]*//g' $wordlist > $oubound 
            ;;

        "-E")
            # Extract all Uppercae Extract all lowercase and uppercase strings from each line and output to wordlist
            sed 's/[^A-Z]*//g' $wordlist > $oubound 
            ;;

        "-Ee")
            # Extract all lowercase and uppercase strings from each line and output to wordlist
            sed 's/[^a-Z]*//g' $wordlist > $oubound 
            ;;

        "-p")
            # Pull 100 random samples from wordlist/password for visual analysis
            shuf -n 100 $wordlist
            ;;
        "-sc")
            # Print statistics on length of each string and total counts per length
            awk '{print length}' $wordlist | sort -n | uniq -c
            ;;
        "-rc")
            # Remove all Duplicate strings and count how many times they are present; then sort by their count by decending order
            sort -nr | uniq -c $wordlist | sort -nr
            ;;

        "-wc")
            # Create custom wordlist based off the website and print all characters that are 5 or greater
            curl -s -k https://$wordlist | grep -oE '[a-z]+' | tr '[:upper:]' '[:lower:]' | sort | uniq | sort -n | grep -E '^.{5,}$' > $outbound
            ;;

        "-rp")
            # remove lines that match from both files and only print lines that have not matched from file2
            grep -vwf -f $wordlist $outbound
            ;;

        "-m")
            # Take your 2 files merge and remove duplicate lines and maintain order
            file1=$wordlist
            file2=$outbound
            outbound="merged_no_duplicates.txt"
            # Remove the output file if it already exists
            rm -f $outbound
            # Concatenate the contents of file1 and file2 into the outbound file
            cat $file1 $file2 >> $outbound
            # Sort the outbound file and remove duplicates
            sort $outbound | uniq > $outbound.tmp
            # Replace the original outbound file with the deduplicated file
            mv $outbound.tmp $outbound
            ;;
Cyrus
  • 84,225
  • 14
  • 89
  • 153
  • 3
    Please take a look at [How to create a **Minimal**, Complete, and Verifiable example](http://stackoverflow.com/help/mcve). – Cyrus Feb 12 '23 at 22:58
  • 1
    Keep a bookmark to https://www.shellcheck.net/. I don't program bash every day myself, but the site helps lint check your program for syntax issues and other potential problems. Has helped me quite often when I get stuck with a script. – Paul T. Feb 12 '23 at 23:00
  • 1
    See the "Before asking about problematic code" and "How to turn a bad script into a good question" sections of the ['bash' tag wiki - Stack Overflow](https://stackoverflow.com/tags/bash/info). – pjh Feb 12 '23 at 23:01
  • 2
    `getopts` can't process multi-character options like `-al`. – Barmar Feb 12 '23 at 23:01
  • See [How can I debug a Bash script?](https://stackoverflow.com/q/951336/4154375). – pjh Feb 12 '23 at 23:02
  • 3
    Why do you have multiple `while getopts` loops? The first one will exit the script for any option other than `-h`. You should just have one loop that sets variables depending on all the options that are given. Then the rest of the script can check the variables. – Barmar Feb 12 '23 at 23:02

1 Answers1

0

As said in the comments, getopts cannot process short options with multiple characters. You also can't execute getopts multiple times because getopts will increase the next option index every time it's invoked.

You can instead use a simple option and argument parsing operation like this:

#!/bin/bash

worldlist= outbound=
shopt -s extglob || exit 1 # Make sure extglob matching is available

usage() {
  echo "Script Syntax: [ bash filemod.sh [-options] wordlist outbound_file]"
  echo "Options:"
  echo "  -h       Show options"
  echo "  -al      Convert alpha characters on each line in file to lowercase characters"
  echo "  -au      Convert alpha characters on each line in file to uppercase characters"
  echo "  -wc      Create custom wordlist based off the website and print all characters that are 5 or greater"
  echo "  -nt      Clean dictionary/wordlist of newlines and tabs"
  echo "  -bj      Clean Dictionary/wordlist of binary data junk/characters left in file"
  echo "  -e       Extract all lowercase strings from each line and output to wordlist"
  echo "  -E       Extract all uppercase strings from each line and output to wordlist"
  echo "  -Ee      Extract all uppercase and lowercase strings from each line and output to wordlist"
  echo "  -el      Extract strings of a specific length into a new file/wordlist"
  echo "  -p       Pull 100 random samples from wordlist/password for visual analysis"
  echo "  -sc      Print statistics on length of each string and total counts per length"
  echo "  -rc      Remove all Duplicate strings and count how many times they are present; then sort by their count by descending order"
  echo "  -rp      Remove lines that match from both files and only print lines that have not matched from file2"
  echo "  -R       Reverse each line in the file from longest to shortest"
  echo "  -x       Split file into separate files by X number of lines per outfile"
  echo "  -m       Take your 2 files merge and remove duplicate lines and maintain order"
  exit 0 # Probably better be a nonzero like 2
}

die() {
  printf '%s\n' "$1" >&2
  exit "${2-1}"
}

extract_lowercase() {
  # Extract all lowercase strings from each line and output to wordlist
  sed 's/[^a-z]*//g' "$wordlist" > "$outbound" || die "Failed to extract lowercase strings."
}

extract_uppercase() {
  # Extract all Uppercae Extract all lowercase and uppercase strings from each line and output to wordlist
  sed 's/[^A-Z]*//g' "$wordlist" > "$outbound" || die "Failed to extract uppercase strings."
}

...

main() {
  local -A opts=() || die "Unable to initialize associative array."
  local names=()

  while [[ $# -gt 0 ]]; do
    case $1 in
    -h)
      usage
      ;;
    -@(al|au|wc|nt|bj|e|E|Ee|el|p|sc|rc|rp|R|x|m)
      opts[${1:1}]=.
      ;;
    --)
      names+=("${@:2}")
      break
      ;;
    -*)
      die "Invalid option: $1"
      ;;
    *)
      names+=("$1")
      ;;
    esac

    shift
  done

  [[ ${#names[@]} -ne 2 ]] && die "Invalid number of names specified."
  [[ ${#opt[@]} -eq 0 ]] && die "No operation option specified."
  [[ ${#opt[@]} -gt 1 ]] && die "Only one option can be specified."

  wordlist=${names} outbound=${names[1]}

  case ${!opts[*]} in
  e) extract_lowercase ;;
  E) extract_uppercase ;;
  ...
  esac
}

main "$@"

The script should also be easy to modify so it can allow multiple operations to happen at once, probably by repeated use of command substitution and heredocs.

konsolebox
  • 72,135
  • 12
  • 99
  • 105