Fix installation errors

README.md

@@ -35,7 +35,7 @@ Then run the following (substituting the actual file you want to scan for `<file
 ``` bash
 $ git clone https://github.com/mitre/multiscanner.git
 $ cd multiscanner
-$ sudo -HE ./install.sh
+$ ./install.sh
 $ multiscanner init
 ```
 

install.sh

@@ -7,20 +7,33 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 #Install requirements for Redhat derivatives
 #Keep these in sync with .travis.yml
 if [ -e /etc/redhat-release ]; then
+sudo bash <<"EOF"
   yum install -y epel-release
   yum install -y autoconf automake curl gcc libffi-devel libtool make python-devel ssdeep-devel tar git unzip openssl-devel file-devel
+  curl -k https://bootstrap.pypa.io/get-pip.py | python
+EOF
 fi
 
 #Install requirements for Debian derivatives
 #Keep these in sync with .travis.yml
 if [ -e /etc/debian_version ]; then
-  apt-get update
-  apt-get install -y build-essential curl dh-autoreconf gcc libffi-dev libfuzzy-dev python-dev git libssl-dev unzip libmagic-dev
+sudo bash <<"EOF"
+  sudo apt-get update
+  sudo apt-get install -y build-essential curl dh-autoreconf gcc libffi-dev libfuzzy-dev python-dev git libssl-dev unzip libmagic-dev
+  sudo curl -k https://bootstrap.pypa.io/get-pip.py | python
+EOF
 fi
 
+# Install the pehash module
+cd ..
+git clone https://github.com/knowmalware/pehash.git
+cd pehash
+python ./setup.py build
+sudo -HE python ./setup.py install
+cd ../multiscanner
+
 # Install multiscanner library and dependencies
-curl -k https://bootstrap.pypa.io/get-pip.py | python
-pip install --upgrade $DIR
+sudo -HE pip install --upgrade $DIR
 
 #Code to compile and install yara
 YARA_VER=3.8.1
@@ -33,10 +46,10 @@ if [[ $prompt == "y" ]]; then
   cd jansson-$JANSSON_VER
   autoreconf -fi
   ./configure --prefix=/usr
-  make install
+  sudo -HE make install
   cd ..
-  rm -rf jansson-$JANSSON_VER
-  ln -s /usr/lib/libjansson.so.4 /lib64/libjansson.so.4
+  sudo rm -rf jansson-$JANSSON_VER
+  sudo ln -s /usr/lib/libjansson.so.4 /lib64/libjansson.so.4
   #We get yara-python as well
   # git clone -b v$YARA_VER https://github.com/VirusTotal/yara-python.git
   curl -L https://github.com/VirusTotal/yara-python/archive/v$YARA_PY_VER.tar.gz | tar -xz
@@ -49,31 +62,32 @@ if [[ $prompt == "y" ]]; then
   grep -q -F 'AC_CONFIG_AUX_DIR([build-aux])' configure.ac || sed -i'' -e 's/AM_INIT_AUTOMAKE/AC_CONFIG_AUX_DIR([build-aux])\
 \
 AM_INIT_AUTOMAKE/g' configure.ac
-  ./bootstrap.sh
+  sudo -HE ./bootstrap.sh
   ./configure --prefix=/usr --enable-magic --enable-cuckoo --enable-dotnet --with-crypto
-  make && make install
+  make && sudo make install
   cd ../
   python setup.py build --dynamic-linking
-  python setup.py install
+  sudo -HE python setup.py install
   cd ../
-  rm -rf yara-python-$YARA_PY_VER
-  ln -s /usr/lib/libyara.so.3 /lib64/libyara.so.3
+  sudo rm -rf yara-python-$YARA_PY_VER
+  sudo ln -s /usr/lib/libyara.so.3 /lib64/libyara.so.3
 fi
 
 read -p "Download yararules.com signatures? <y/N> " prompt
 if [[ $prompt == "y" ]]; then
-  git clone --depth 1 https://github.com/Yara-Rules/rules.git ~/.multiscanner/yarasigs/Yara-Rules
-  echo You can update these signatures by running cd ~/.multiscanner/yarasigs/Yara-Rules \&\& git pull
+  git clone --depth 1 https://github.com/Yara-Rules/rules.git ~/.multiscanner/etc/yarasigs/Yara-Rules
+  echo You can update these signatures by running cd ~/.multiscanner/etc/yarasigs/Yara-Rules \&\& git pull
 fi
 
 read -p "Download SupportIntelligence's Icewater yara signatures? <y/N> " prompt
 if [[ $prompt == "y" ]]; then
-  git clone --depth 1 https://github.com/SupportIntelligence/Icewater.git ~/.multiscanner/yarasigs/Icewater
-  echo You can update these signatures by running cd ~/.multiscanner/yarasigs/Icewater \&\& git pull
+  git clone --depth 1 https://github.com/SupportIntelligence/Icewater.git ~/.multiscanner/etc/yarasigs/Icewater
+  echo You can update these signatures by running cd ~/.multiscanner/etc/yarasigs/Icewater \&\& git pull
 fi
 
 read -p "Download TrID? <y/N> " prompt
 if [[ $prompt == "y" ]]; then
+sudo bash <<"EOF"
   mkdir -p /opt/trid
   cd /opt/trid
   curl -f --retry 3 http://mark0.net/download/trid_linux_64.zip > trid.zip
@@ -92,23 +106,29 @@ if [[ $prompt == "y" ]]; then
   rm -f triddefs.zip
   chmod 755 trid
   cd $CWD
+EOF
 fi
 
 read -p "Download FLOSS? <y/N> " prompt
 if [[ $prompt == "y" ]]; then
+sudo bash <<"EOF"
   curl -f --retry 3 https://s3.amazonaws.com/build-artifacts.floss.flare.fireeye.com/travis/linux/dist/floss > /opt/floss
   chmod 755 /opt/floss
+EOF
 fi
 
 read -p "Download NSRL database? This will take ~4GB of disk space. <y/N> " prompt
 if [[ $prompt == "y" ]]; then
   # Download the unique set
-  mkdir ~/.multiscanner/nsrl
+  mkdir -p ~/.multiscanner/etc/nsrl
+  mkdir ~/nsrl_tmp
+  cd ~/nsrl_tmp
   curl -k https://s3.amazonaws.com/rds.nsrl.nist.gov/RDS/current/rds_modernu.zip > rds_modernu.zip
   unzip rds_modernu.zip
   rm rds_modernu.zip
-  python $DIR/multiscanner/utils/nsrl_parse.py -o ~/.multiscanner/nsrl RDS_*/NSRLFile.txt
-  rm -fr RDS_*
+  python $DIR/multiscanner/utils/nsrl_parse.py -o ~/.multiscanner/etc/nsrl NSRLFile.txt
+  cd -
+  rm -fr ~/nsrl_tmp
 fi
 
 # Initialize multiscanner

requirements.txt

@@ -3,6 +3,7 @@ future
 six
 configparser
 click==6.7
+pathlib
 #Required by modules
 bitstring
 boltons
@@ -28,11 +29,14 @@ sqlalchemy
 sqlalchemy-utils
 sqlalchemy-datatables
 #Required by storage modules
-elasticsearch>=6.0.0,<7.0.0
+urllib3<1.25,>=1.24.2
+elasticsearch>=7.0.0,<8.0.0
 elasticsearch-curator
 pymongo
 #Required for distributed
 celery
 flask_cors
 #Required by utils
+mmbot
+lightgbm
 tqdm