Installation script for Mail Piler (open source version) on Ubuntu 20.04 Focal

focal-master.sh

#!/bin/bash
#
#
# This script installs the latest open source version of Mail Piler (mailpiler.org) from the master
# branch on Bitbucket by compiling it from source. It also installs all dependencies, including a MySQL database.
#
# You should run this script as root on a vanilla Ubuntu 20.04 installation.
#
#
set -o errexit
set -o pipefail
set -o nounset
set -x

PILER_HOSTNAME="${PILER_HOSTNAME:-archive.yourdomain.com}"
MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
SERVER_ID="${SERVER_ID:-0}"
USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
PHP_FPM_SOCKET="/var/run/php/php7.4-fpm.sock"

MYSQL_HOSTNAME="localhost"
MYSQL_DATABASE="piler"
MYSQL_USERNAME="piler"

SPHINX_TARGZ="sphinx-3.3.1-bin.tar.gz"
DOWNLOAD_URL="https://download.mailpiler.com"
PILER_TARBALL="https://bitbucket.org/jsuto/piler/get/master.tar.gz"
PILER_USER="piler"
CONFIG_SITE_PHP="/etc/piler/config-site.php"
CONFIG_SITE_DIST_PHP="/etc/piler/config-site.dist.php"

export DEBIAN_FRONTEND=noninteractive

install_prerequisites() {
   apt-get update
   apt-get -y --no-install-recommends install \
      wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
      php7.4-gd php7.4-curl php7.4-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip5 \
      libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client-core-10.3 \
      mariadb-server-10.3 build-essential libssl-dev libtre-dev libzip-dev

   wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}"
   tar -C / -zxvf "/tmp/${SPHINX_TARGZ}"
}

create_user() {
   egrep -i "^x$PILER_USER:" /etc/passwd || adduser --no-create-home --disabled-password --disabled-login --gecos "" $PILER_USER
}

create_mysql_user_and_database() {
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "CREATE USER IF NOT EXISTS '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "SET PASSWORD FOR '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME' = PASSWORD('$MYSQL_PILER_PASSWORD');"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "CREATE DATABASE IF NOT EXISTS $MYSQL_DATABASE CHARACTER SET 'utf8mb4';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "GRANT ALL PRIVILEGES ON $MYSQL_DATABASE.* to '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME' IDENTIFIED BY '$MYSQL_PILER_PASSWORD';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "FLUSH PRIVILEGES;"

   mysql -u $MYSQL_USERNAME -p"$MYSQL_PILER_PASSWORD" $MYSQL_DATABASE < /etc/piler/db-mysql.sql
}

fix_mysql_settings() {
   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
[mysqld]

innodb_buffer_pool_size=512M
innodb_flush_log_at_trx_commit=1
innodb_log_buffer_size=64M
innodb_log_file_size=64M
innodb_read_io_threads=4
innodb_write_io_threads=4
innodb_log_files_in_group=2

innodb_file_per_table
PILER_CNF
}

start_mysql() {
   fix_mysql_settings
   service mysql restart
}

install_piler() {
   wget "${PILER_TARBALL}" -O "/tmp/piler.tar.gz"
   tar -zxvf "/tmp/piler.tar.gz" -C /tmp
   pushd /tmp/jsuto-piler-*
   ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb --enable-tcpwrappers --enable-memcached
   make clean all install
   cp ./contrib/webserver/piler-nginx.conf /etc/piler/piler-nginx.conf.dist
   cp ./etc/sphinx.conf.dist /etc/piler/sphinx.conf
   cp ./util/db-mysql.sql /etc/piler/db-mysql.sql
   popd

   crontab -u "$PILER_USER" /usr/share/piler/piler.cron

   touch /var/piler/.bash_history
   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
}

create_my_cnf() {
   local user=$1
   local password=$2
   local my_cnf=$3

   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
   chown $PILER_USER:$PILER_USER "$my_cnf"
   chmod 600 "$my_cnf"
}


fix_config_site_php() {
   cp $CONFIG_SITE_DIST_PHP $CONFIG_SITE_PHP
   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"

   {
      echo "\$config['SERVER_ID'] = $SERVER_ID;"
      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
      echo "\$config['SPHINX_VERSION'] = 331;"
   } >> "$CONFIG_SITE_PHP"

   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
   fi

   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
}


add_systemd_services() {
   pushd /etc/systemd/system

   ln -sf /usr/libexec/piler/piler.service .
   ln -sf /usr/libexec/piler/piler-smtp.service .
   ln -sf /usr/libexec/piler/pilersearch.service .

   popd

   systemctl daemon-reload

   systemctl enable piler
   systemctl enable piler-smtp
   systemctl enable pilersearch
}

create_cipher_key() {
   dd if=/dev/urandom bs=56 count=1 of=/etc/piler/piler.key
   chmod 640 /etc/piler/piler.key
   chown piler:piler /etc/piler/piler.key
}


fix_configs() {
   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
      ln -s /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf

      nginx -t
      nginx -s reload
   fi

   if [[ ! -f /etc/piler/piler.conf ]]; then
      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.yourdomain.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
      chmod 600 /etc/piler/piler.conf
      chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
   fi

   sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \
          -e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \
          -e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \
          -e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \
          /etc/piler/sphinx.conf
}

install_prerequisites
create_user
install_piler
create_mysql_user_and_database
start_mysql

create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf

fix_configs
fix_config_site_php

add_systemd_services
create_cipher_key

su -c "indexer --all -c /etc/piler/sphinx.conf" $PILER_USER

[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler

systemctl start pilersearch
systemctl start piler
systemctl start piler-smtp

Hi, Thanks for your script.
I'm trying to use it but it's not working and I think it's because Piler now uses Manticore and no Sphinx.
Could you update for Manticore (I'm too newbie to do it! Sorry). Or backport it to Sphinx?

Regards,

Jáder

Hi, Thanks for your script. I'm trying to use it but it's not working and I think it's because Piler now uses Manticore and no Sphinx. Could you update for Manticore (I'm too newbie to do it! Sorry). Or backport it to Sphinx?

Regards,

Jáder

Hi, One question were you able to resolve the installation in Ubuntu 20.04?, I am in the same situation.

No I haven't made it work. Sorry. Moved to try on DOCKER... for now it's working that way. Jáder Marasca +55.83.999.899.232 (Tim WhatsApp) +55.51.2160.8383 (VoIP) Em qui., 8 de fev. de 2024 às 15:09, rickygm ***@***.***> escreveu:

…

***@***.**** commented on this gist. ------------------------------ Hi, Thanks for your script. I'm trying to use it but it's not working and I think it's because Piler now uses Manticore and no Sphinx. Could you update for Manticore (I'm too newbie to do it! Sorry). Or backport it to Sphinx? Regards, Jáder Hi, One question were you able to resolve the installation in Ubuntu 20.04?, I am in the same situation. — Reply to this email directly, view it on GitHub <https://gist.github.com/cmer/08d80990f0c87e535e5d4904650f8326#gistcomment-4884235> or unsubscribe <https://github.com/notifications/unsubscribe-auth/AEBNCRLQNOTP3CAVIRFMCNDYSUIFZBFKMF2HI4TJMJ2XIZLTSKBKK5TBNR2WLJDUOJ2WLJDOMFWWLO3UNBZGKYLEL5YGC4TUNFRWS4DBNZ2F6YLDORUXM2LUPGBKK5TBNR2WLJDHNFZXJJDOMFWWLK3UNBZGKYLEL52HS4DFVRZXKYTKMVRXIX3UPFYGLK2HNFZXIQ3PNVWWK3TUUZ2G64DJMNZZDAVEOR4XAZNEM5UXG5FFOZQWY5LFVEYTCNBSHA3DOMZXU52HE2LHM5SXFJTDOJSWC5DF> . You are receiving this email because you commented on the thread. Triage notifications on the go with GitHub Mobile for iOS <https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675> or Android <https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub> .

Could you share how you did it with Docker?

For Anyone coming here in the future, here is what you would need to do to use manticore instead of sphinx:

Remove these lines from script above:

wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}"
tar -C / -zxvf "/tmp/${SPHINX_TARGZ}"

Assuming Ubuntu 22.04: install manticore:
wget https://repo.manticoresearch.com/manticore-repo.noarch.deb
sudo dpkg -i manticore-repo.noarch.deb
sudo apt update
sudo apt install manticore manticore-extra

Change line 172 from /etc/piler/sphinx.conf to /etc/piler/manticore.conf

run this command to let piler know to use manticore: touch /etc/piler/MANTICORE

That should be all that's needed. for reference, also check out this link : https://www.mailpiler.org/manticore-search/

Hope this helps someone