Skip to content

Instantly share code, notes, and snippets.

@linoskoczek
Last active August 2, 2022 13:28
Show Gist options
  • Save linoskoczek/6e1d5b569f700d078305a6e963e44113 to your computer and use it in GitHub Desktop.
Save linoskoczek/6e1d5b569f700d078305a6e963e44113 to your computer and use it in GitHub Desktop.
A script to set up a local CodeQL environment (or update if already present) and scan it using Security queries. Results are saved in CSV files and merged into a single CSV afterwards. Empty results are removed. Before running, edit SOURCE_ROOT, PLANG and PLATFORM variables. Tested only on Linux.
#!/bin/bash
# VARIABLES TO EDIT
SOURCE_ROOT="/home/path/to/the/code/that/you/want/to/scan" # might be a good idea to remove all special characters from the name
PLANG="csharp" # cpp, csharp, go, java, javascript, python, ruby
PLATFORM="linux64" # linux64, osx64, win64
# VARIABLES NOT TO EDIT
DATABASE_NAME="${SOURCE_ROOT##*/}_db"
# cd TO SCRIPT DIR
SCRIPT_RELATIVE_DIR=$(dirname "${BASH_SOURCE[0]}")
cd $SCRIPT_RELATIVE_DIR
echo $SCRIPT_RELATIVE_DIR
# DOWNLOAD/UPDATE CODEQL
echo -e "> CHECKING FOR CodeQL PRESENCE"
CODEQL_BINARY_ZIP_NAME="codeql-$PLATFORM.zip"
if ( test -f $CODEQL_BINARY_ZIP_NAME ) then
SIZE_OF_DOWNLOADED_CODEQL_RELEASE=$(stat --format="%s" $CODEQL_BINARY_ZIP_NAME)
else
SIZE_OF_DOWNLOADED_CODEQL_RELEASE=0
fi
SIZE_OF_NEWEST_CODEQL_RELEASE=$(curl -s https://api.github.com/repos/github/codeql-cli-binaries/releases | grep '"size"' | head -n 1 | cut -d':' -f2 | tr -d ' ' | tr -d ',')
if [ $SIZE_OF_DOWNLOADED_CODEQL_RELEASE -ne $SIZE_OF_NEWEST_CODEQL_RELEASE ]
then
echo "Downloading newest CodeQL binary."
rm -f $CODEQL_BINARY_ZIP_NAME
rm -rf codeql
wget -O $CODEQL_BINARY_ZIP_NAME $(curl -s https://api.github.com/repos/github/codeql-cli-binaries/releases | grep 'browser_' | grep $PLATFORM | head -n 1 | cut -d\" -f4)
unzip $CODEQL_BINARY_ZIP_NAME
else
echo "CodeQL is already newest version."
fi
echo -e "\n> LIST OF SUPPORTED LANGUAGES (checking if CodeQL works)"
./codeql/codeql resolve languages
echo -e "\n> CREATING CODEQL-HOME (downloading queries)"
git clone --depth=1 --recurse-submodules -j8 https://github.com/github/codeql codeql-home
cd codeql-home
git pull --recurse-submodules
cd ..
# CREATE CodeQL DATABASE
echo -e "\n> CREATING DATABASE"
mkdir -p databases
rm -rf ./databases/$DATABASE_NAME
./codeql/codeql database create ./databases/$DATABASE_NAME --language=$PLANG --source-root $SOURCE_ROOT
if [ $? -eq 0 ]; then
echo "Database created successfully"
else
echo "Database was not created. Cannot go further."
echo "Read the above errors. If you used SonarQube, it might be helpful to remove .sonarqube directory."
exit
fi
echo -e "\n> REMOVING SPACES FROM CODEQL QUERY DIRECTORIES"
mv "./codeql-home/$PLANG/ql/src/Security Features/" "./codeql-home/$PLANG/ql/src/SecurityFeatures/"
mv "./codeql-home/$PLANG/ql/src/Input Validation/" "./codeql-home/$PLANG/ql/src/InputValidation/"
# ANALYZE DATABASE
OUTPUT_DIR="$(date +"%Y-%m-%d-%H-%M")-${DATABASE_NAME}-output"
mkdir -p $OUTPUT_DIR
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")
echo -e "\n> ANALYZING DATABASE WITH $PLANG SECURITY QUERIES"
for dir in ./codeql-home/$PLANG/ql/src/SecurityFeatures/*
do
dir=${dir%*/}
single="${dir##*/}"
output_file="./$OUTPUT_DIR/$single.csv"
./codeql/codeql database analyze ./databases/$DATABASE_NAME --format=CSV --output=$output_file --threads 0 $dir
if test -f "$output_file"; then
single_output_size=$(stat --format="%s" $output_file)
if [ "$single_output_size" == "0" ]
then
echo "Output has 0 bytes so removing it..."
rm -f $output_file
else
echo "Output saved to $output_file"
fi
else
echo "No output - probably an error with this single query."
fi
done
echo -e "\n> ANALYZING DATABASE WITH $PLANG INPUT VALIDATION"
output_file="./$OUTPUT_DIR/$single.csv"
./codeql/codeql database analyze ./databases/$DATABASE_NAME --format=CSV --output=$output_file --threads 0 "./codeql-home/$PLANG/ql/src/InputValidation/"
if test -f "$output_file"; then
single_output_size=$(stat --format="%s" $output_file)
if [ "$single_output_size" == "0" ]
then
echo "Output has 0 bytes so removing it..."
rm -f $output_file
else
echo "Output saved to $output_file"
fi
else
echo "No output - probably an error with this single query."
fi
IFS=$SAVEIFS
echo -e "\n> COMBINING RESULTS INTO all.csv FILE"
cat $(pwd)/$OUTPUT_DIR/* > $(pwd)/$OUTPUT_DIR/all.csv
echo -e "\n> DONE! See results here: file://$(pwd)/$OUTPUT_DIR/"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment