README_PRODUCTION_INSTALL

#
#  Installation of a web server. [JPH 20150704]
#
#  $IP_ADDR            ip address of new server
#  $USER               jason
#  $USER_PASSWORD      xxx
#  $DATA_PASSWORD      xxx
#  $MAIL_PASSWORD      xxx
#
################################################################################

local> ssh root@$IP_ADDR
root> passwd -l root

# Create account for webserver.
root> useradd -m -s /bin/bash mo
root> passwd -l mo
root> visudo
  # Add these lines to end. Allows users in "mo" group to "sudo su mo".
  %mo ALL = NOPASSWD: /bin/su mo
  %mo ALL = NOPASSWD: /bin/su - mo
  mo ALL = NOPASSWD: /usr/sbin/service nginx *
  mo ALL = NOPASSWD: /usr/sbin/service puma *

# Create user accounts.
root> useradd -m -s /bin/bash -G mo,sudo jason
root> useradd -m -s /bin/bash -G mo,sudo jdcohenesq
root> useradd -m -s /bin/bash -G mo,sudo alan
root> useradd -m -s /bin/bash -G mo,sudo andrnimm
root> mkdir /home/jason/.ssh
root> mkdir /home/jdcohenesq/.ssh
root> mkdir /home/alan/.ssh
root> mkdir /home/andrnimm/.ssh
root> vi /home/jason/.ssh/authorized_keys
  # copy over from old server
root> vi /home/jdcohenesq/.ssh/authorized_keys
  # copy over from old server
root> vi /home/alan/.ssh/authorized_keys
  # copy over from old server
root> vi /home/andrnimm/.ssh/authorized_keys
  # copy over from old server
root> chmod 600 /home/jason/.ssh/authorized_keys
root> chmod 600 /home/jdcohenesq/.ssh/authorized_keys
root> chmod 600 /home/alan/.ssh/authorized_keys
root> chmod 600 /home/andrnimm/.ssh/authorized_keys
root> chmod 700 /home/jason/.ssh
root> chmod 700 /home/jdcohenesq/.ssh
root> chmod 700 /home/alan/.ssh
root> chmod 700 /home/andrnimm/.ssh
root> chown -R jason:jason /home/jason/.ssh
root> chown -R jdcohenesq:jdcohenesq /home/jdcohenesq/.ssh
root> chown -R alan:alan /home/alan/.ssh
root> chown -R andrnimm:andrnimm /home/andrnimm/.ssh
root> passwd $USER
  # set your password now just to be safe, in case keyless login fails
root> logout

# Set up your account.
local> ssh $IP_ADDR
user> vi .bash_aliases
  # Add this at least.
  alias mo='cd /var/web/mushroom-observer && sudo su mo'
user> sudo su

# Set up root account.
root> vi /root/.bashrc
  # Add these at end.
  export CFLAGS="-O3 -m64"
  alias mo='cd /var/web/mushroom-observer && su mo'
root> source /root/.bashrc
root> vi /etc/ssh/sshd_config
  PermitRootLogin no

# Install core packages.
root> apt-get update
root> apt-get upgrade -y
root> apt-get install -y telnet wget curl vim iptables mlocate lynx ntp

# Configure firewall. (22=ssh, 80=http, 25=smtp, 3306=mysql, 53=dns)
# Run "sudo iptables-save" on old server, then copy into here.
root> iptables-restore [copy of firewall from old server]
root> iptables-save > /etc/firewall.conf
root> chmod 600 /etc/firewall.conf
# What does the following do????
root> vi /etc/rc.local
  iptables-restore < /etc/firewall.conf
  swapon /swap

# Create swap file if not one already.
# (I'm moving this ahead because it was hanging during installation
# of the gems, and I think it migh have been because the droplet was
# too small, because when I added swap it suddenly completed.  So
# if it really did need swap to perform the installation, the sooner
# we add swap the better, right? -JPH 20230813)
root> free -h
# If there is no swap do this:
root> fallocate -l 1G /swap
root> chmod 600 /swap
root> mkswap /swap
root> swapon /swap
root> echo '/swap none swap sw 0 0' | tee -a /etc/fstab

# Install most of the modules that we will need (except ruby).
root> apt-get install -y mysql-server mysql-client libmysqlclient-dev \
          libcurl4-openssl-dev libssl-dev git nginx-extras libyaml-dev \
          imagemagick libmagickcore-dev libmagickwand-dev libjpeg-dev \
          exiftool mrtg
# (it's saying several services need to be restarted so rebooting
# sounded like a good idea at this point)
root> reboot

# Install rvm and ruby. [CHECK LATEST RUBY VERSION!!!]
root> curl -sSL https://rvm.io/mpapis.asc | sudo gpg --import -
root> curl -sSL https://get.rvm.io | bash -s stable --ruby=3.1.2
# It failed here, so I followed the instructions and reran the cmd:
# (I don't understand the following cmd, so it may change?)
root> sudo gpg2 --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
root> curl -sSL https://get.rvm.io | bash -s stable --ruby=3.1.2
root> gpasswd -a mo rvm
root> echo "gem: --no-document" > /etc/gemrc
root> echo "gem: --no-document" > /root/.gemrc
root> echo "gem: --no-document" > /home/mo/.gemrc
root> echo "source /usr/local/rvm/scripts/rvm" >> /root/.bashrc
root> echo "source /usr/local/rvm/scripts/rvm" >> /home/mo/.bashrc

# Create directory for rails app.
root> mkdir /var/web
root> mkdir /var/web/mo
root> chown mo:mo /var/web/mo
root> chmod 750 /var/web/mo
root> ln -s /var/web/mo /var/web/mushroom-observer

# Make sure nginx user has access to rails directories.
root> usermod -a -G mo www-data

# Install and configure rails apps.
root> cd /var/web/mushroom-observer
root> su mo
mo> git clone https://github.com/MushroomObserver/mushroom-observer .
[NJW: Remember to switch to another branch if you are testing this process
on code not on the main branch.]

# If trying to keep a completely fresh installation:
mo> mkdir log log/old tmp tmp/pids tmp/sockets
mo> mkdir public/images/{thumb,320,640,960,1280,orig}
mo> ln -s images public/local_images
mo> cp config/etc/bash_aliases.sh ~/.bash_aliases
mo> vi config/consts-site.rb
  # grab this from old server
mo> vi config/database.yml
  # grab this from old server
mo> echo blahblahblah > config/master.key
  # grab "blahblahblah" from the old server
# Ping the database to make sure everything's working right.
# (You will have to add the new IP address to the database server!)
database_server_root> mysql -u root -p
  # For each remote user:
  CREATE USER 'mo'@'$WEB_SERVER' IDENTIFIED BY '$DB_PASSWORD';
  GRANT ALL PRIVILEGES ON mo_production.* TO 'mo'@'$WEB_SERVER' WITH GRANT OPTION;
  GRANT PROCESS, SELECT, LOCK TABLES ON *.* TO 'mo'@'$WEB_SERVER';
database_server_root> service mysql restart
# Now test it on the new server...
mo> mysqladmin --protocol=tcp -u mo -p'$DB_PASSWORD' --host=db3.mushroomobserver.org ping

# Or can just mirror old server like this:
mo> cd /var/web/mo
mo> rsync -av jason@$OLD_SERVER_IP:/var/web/mo/ .

# Install gems.
mo> gem install bundle
mo> bundle install

# Configure nginx.
mo> exit
root> cp /var/web/mushroom-observer/config/etc/nginx.conf /etc/nginx/
# This is failing because of letsencrypt not being installed yet.
# root> apt install snapd
# root> snap install --classic certbot
# root> ln -s /snap/bin/certbot /usr/bin/certbot
# root> certbot --nginx
root> service nginx restart
root> rm /usr/share/nginx/html/index.html # (there's *got* to be a better way!)

# Install our programs for resizing and rotating JPEG images.
root> gcc /var/web/mushroom-observer/script/jpegresize.c -ljpeg -lm -O2 -o /usr/local/bin/jpegresize
root> cp /var/web/mushroom-observer/script/exifautotran /usr/local/bin/exifautotran
root> chmod 755 /usr/local/bin/exifautotran

root> cp /var/web/mushroom-observer/config/etc/puma.service /etc/systemd/system/
root> chmod 755 /etc/systemd/system/puma.service
root> systemctl enable puma

root> cp /var/web/mushroom-observer/config/etc/solidqueue.service /etc/systemd/system/
root> chmod 755 /etc/systemd/system/solidqueue.service
root> systemctl enable solidqueue

--------------------------------------------------------------------------------

# If installing database locally:
root> mysql -u root -p
  CREATE DATABASE mo_production;
  CREATE USER 'mo'@'localhost' IDENTIFIED BY '$DB_PASSWORD';
  GRANT ALL PRIVILEGES ON mo_production.* TO 'mo'@'localhost' WITH GRANT OPTION;
  EXIT
root> scp $SNAPSHOT_PATH/checkpoint.gz .
root> gunzip -cd checkpoint.gz | mysql -u mo -p'$DB_PASSWORD' mo_production

--------------------------------------------------------------------------------

# Test basic system now.
root> su mo
mo> env | grep RAILS_ENV
# Should print "production"!!
mo> bundle install
mo> rake db:migrate
mo> rake lang:update
mo> rake assets:precompile
mo> rails server
# Watch for errors in any of these!!

mo> puma
check for errors on stdout/err

# Should start listening on port 8080.
# This isn't working, also seems to be an ssl issue?
mo> wget localhost:8080

--------------------------------------------------------------------------------

# Set up simple mailserver using postfix.
mo> exit
root> apt-get install -y postfix mutt
  # Select "internet site" when it prompts you.
  # Change hostname to "mushroomobserver.org" (should resolve back with reverse DNS).
root> vi /etc/aliases
  # Change where root's mail goes.
  root: $USER,...
root> newaliases

# NOTE: After moving mail.mo.org to gmail, had to tweak postfix configuration:
root> vi /etc/postfix/main.cf
  # change myhostname to "mushroomobserver.org" (instead of "xxx.mushroomobserver.org")
[NJW: Not needed?  Was already set to 'ror30.mushroomobserver.org']
[JPH: I'm not sure what the purpose of this was.  Might be an RDNS thing.]

# Test mail server.
root> su $USER
user> cd
user> echo $YOUR_EMAIL > .forward
user> vi .muttrc
  set realname="$FULL_NAME"
  set from="$USER@mushroomobserver.org"
  set use_from=yes
  set envelope_from=yes
user> echo "This is a test." | mutt -s test $YOUR_EMAIL
  # (Check /var/log/mail.log and /var/log/mail.err.)
user> exit

# Configure mail for MO.
root> su mo
mo> cd
mo> vi .muttrc
  set realname="Mushroom Observer"
  set from="no-reply@mushroomobserver.org"
  set use_from=yes
  set envelope_from=yes
mo> echo root > .forward
mo> echo "This is another test." | mutt -s test $YOUR_EMAIL
mo> exit

# Configure mail for rails.
# Should come configured "out of the box".
# Ask yourself a question via the website, e.g.:
#   http://$IP_ADDR/observer/ask_user_question/252
# Probably want to tail -f log/production.log if it's throwing errors.

# Configure no-reply to bounce correctly.
root> useradd -m no-reply
root> passwd -l no-reply
root> cp /var/web/mushroom-observer/script/autoreply              /usr/local/bin/autoreply
root> cp /var/web/mushroom-observer/config/etc/no-reply.muttrc    /home/no-reply/.muttrc
root> cp /var/web/mushroom-observer/config/etc/no-reply.autoreply /home/no-reply/.autoreply
root> cp /var/web/mushroom-observer/config/etc/no-reply.forward   /home/no-reply/.forward
root> chmod 644 /home/no-reply/.[maf]*
root> chmod 755 /usr/local/bin/autoreply

# So we always run into problems with being black-listed by spam filters after
# we change IP address.  I don't know how to fix this.  But one thing to check
# is the magic "TXT" record in the DNS.  Also, check for bounced mail in
# /var/mail/news.

--------------------------------------------------------------------------------

# Set up password-less ssh for image transfers.
mo> su mo
mo> ssh-keygen -t rsa
  # Just press return three times, tell it to overwrite id_rsa.pub if it asks.
  # Now copy and paste
  #   /home/mo/.ssh/id_rsa.pub
  # onto the end of
  #   /home/mo/.ssh/authorized_keys
  # on the image server.  Test if it worked:
mo> ssh images.mushroomobserver.org echo hello world

# Set up logrotate to archive puma and rails logs.
mo> exit
root> vi /etc/logrotate.d/puma
  /var/web/mushroom-observer/log/*.log {
    daily
    missingok
    rotate 7
    dateext
    compress
    delaycompress
    sharedscripts
    postrotate
      kill -HUP `cat /var/web/mushroom-observer/tmp/pids/puma.pid`
      mv /var/web/mushroom-observer/log/*.gz /var/web/mushroom-observer/log/old/
    endscript
  }
root> su mo

# Install cron jobs.
mo> crontab /var/web/mushroom-observer/config/etc/crontab

--------------------------------------------------------------------------------

# Setup mrtg account.
root> vi /etc/logrotate.d/nginx
  # change create line from "640" to:
  create 0644 www-data adm
root> cp /var/web/mo/config/etc/indexmaker /usr/bin/
root> chmod 644 /var/log/nginx/access.log
root> useradd -m -s /bin/bash -G mo mrtg
root> su mrtg
mrtg> cd
mrtg> ssh-keygen -t rsa
  # Press return three times.
mrtg> cat .ssh/id_rsa.pub
  # Copy this into old server's /home/mrtg/.ssh/authorized_keys.
mrtg> rsync $OLD_SERVER_IP:/home/mrtg/ .
mrtg> crontab -e

# MRTG uses a script in alan's home directory, so I rsynced alan's home, too.

--------------------------------------------------------------------------------

# A few post-installation sanity-checks.  Reboot after everything is done,
# then check the following.

# Make sure firewall is working.  If there are more than a few lines it is
# probably correct, but you can also compare it with another server.
root> iptables-save

# Make sure all the critical processes started automatically.
# Look for sshd, nginx, puma, mysqld, postfix.
root> ps -ef | grep sshd
root> ps -ef | grep nginx
root> ps -ef | grep puma
root> ps -ef | grep mysql
root> ps -ef | grep postfix

# To enable or disable startup of a service:
root> systemctl enable|disable $SERVICE

# Make sure date and time and timezone are correct!
root> date
root> TZ='America/Los_Angeles' date

# Make sure MRTG is running.
browser> http:$NEW_SERVER_IP/stats/index.html