2021-11-02 14:45:05 -07:00
|
|
|
---
|
2021-11-02 16:29:52 -07:00
|
|
|
- name: Set up the environment for the impress-2020 app
|
2021-11-02 16:01:30 -07:00
|
|
|
hosts: webserver
|
2021-11-03 01:00:28 -07:00
|
|
|
vars:
|
|
|
|
email_address: "emi@matchu.dev" # TODO: Extract this to personal config?
|
2021-11-02 14:45:05 -07:00
|
|
|
tasks:
|
2021-11-12 20:14:44 -08:00
|
|
|
- name: Copy authorized SSH keys
|
|
|
|
copy:
|
|
|
|
dest: ~/.ssh/authorized_keys
|
|
|
|
src: ../authorized-ssh-keys.txt
|
|
|
|
|
2021-11-03 23:04:25 -07:00
|
|
|
- name: Disable root SSH login
|
|
|
|
become: yes
|
|
|
|
lineinfile:
|
|
|
|
dest: /etc/ssh/sshd_config
|
|
|
|
regexp: ^#?PermitRootLogin
|
|
|
|
line: PermitRootLogin no
|
|
|
|
|
|
|
|
- name: Disable password-based SSH authentication
|
|
|
|
become: yes
|
|
|
|
lineinfile:
|
|
|
|
dest: /etc/ssh/sshd_config
|
|
|
|
regexp: ^#?PasswordAuthentication
|
|
|
|
line: PasswordAuthentication no
|
|
|
|
|
|
|
|
- name: Install fail2ban firewall with default settings
|
|
|
|
become: yes
|
|
|
|
apt:
|
|
|
|
update_cache: yes
|
|
|
|
name: fail2ban
|
|
|
|
|
2021-11-04 18:57:00 -07:00
|
|
|
- name: Configure ufw firewall to allow SSH connections on port 22
|
|
|
|
become: yes
|
|
|
|
community.general.ufw:
|
|
|
|
rule: allow
|
|
|
|
port: "22"
|
|
|
|
|
|
|
|
- name: Configure ufw firewall to allow HTTP connections on port 80
|
|
|
|
become: yes
|
|
|
|
community.general.ufw:
|
|
|
|
rule: allow
|
|
|
|
port: "80"
|
|
|
|
|
|
|
|
- name: Configure ufw firewall to allow HTTP connections on port 443
|
|
|
|
become: yes
|
|
|
|
community.general.ufw:
|
|
|
|
rule: allow
|
|
|
|
port: "443"
|
|
|
|
|
|
|
|
- name: Enable ufw firewall with all other ports closed by default
|
|
|
|
become: yes
|
|
|
|
community.general.ufw:
|
|
|
|
state: enabled
|
|
|
|
policy: deny
|
|
|
|
|
2021-11-04 19:17:35 -07:00
|
|
|
- name: Install unattended-upgrades
|
|
|
|
become: yes
|
|
|
|
apt:
|
|
|
|
update_cache: yes
|
|
|
|
name: unattended-upgrades
|
|
|
|
|
|
|
|
- name: Enable unattended-upgrades to auto-upgrade our system
|
|
|
|
become: yes
|
|
|
|
copy:
|
|
|
|
content: |
|
|
|
|
APT::Periodic::Update-Package-Lists "1";
|
|
|
|
APT::Periodic::Unattended-Upgrade "1";
|
|
|
|
dest: /etc/apt/apt.conf.d/20auto-upgrades
|
|
|
|
|
|
|
|
- name: Configure unattended-upgrades to auto-reboot our server when necessary
|
|
|
|
become: yes
|
|
|
|
lineinfile:
|
|
|
|
regex: ^(//\s*)?Unattended-Upgrade::Automatic-Reboot ".*";$
|
|
|
|
line: Unattended-Upgrade::Automatic-Reboot "true";
|
|
|
|
dest: /etc/apt/apt.conf.d/50unattended-upgrades
|
|
|
|
|
|
|
|
- name: Configure unattended-upgrades to delay necessary reboots to 3am
|
|
|
|
become: yes
|
|
|
|
lineinfile:
|
|
|
|
regex: ^(//\s*)?Unattended-Upgrade::Automatic-Reboot-Time ".*";$
|
|
|
|
line: Unattended-Upgrade::Automatic-Reboot-Time "03:00";
|
|
|
|
dest: /etc/apt/apt.conf.d/50unattended-upgrades
|
|
|
|
|
|
|
|
- name: Configure the system timezone to be US Pacific time
|
|
|
|
become: yes
|
|
|
|
community.general.timezone:
|
|
|
|
name: America/Los_Angeles
|
|
|
|
|
2021-11-03 23:04:25 -07:00
|
|
|
- name: Create the app versions folder
|
2021-11-02 16:01:30 -07:00
|
|
|
become: yes
|
|
|
|
file:
|
2021-11-03 23:04:25 -07:00
|
|
|
path: /srv/impress-2020/versions
|
2021-11-03 16:59:23 -07:00
|
|
|
owner: "{{ ansible_user_id }}"
|
2021-11-03 23:04:25 -07:00
|
|
|
group: "{{ ansible_user_id }}"
|
|
|
|
state: directory
|
2021-11-02 18:47:13 -07:00
|
|
|
|
2021-11-02 16:01:30 -07:00
|
|
|
- name: Add Nodesource apt key
|
|
|
|
become: yes
|
|
|
|
apt_key:
|
|
|
|
id: 9FD3B784BC1C6FC31A8A0A1C1655A0AB68576280
|
|
|
|
url: https://deb.nodesource.com/gpgkey/nodesource.gpg.key
|
2021-11-02 18:47:13 -07:00
|
|
|
|
2021-11-02 16:01:30 -07:00
|
|
|
- name: Add Node v16 apt repository
|
|
|
|
become: yes
|
|
|
|
apt_repository:
|
|
|
|
repo: deb https://deb.nodesource.com/node_16.x focal main
|
2021-11-02 18:47:13 -07:00
|
|
|
|
2021-11-02 16:01:30 -07:00
|
|
|
- name: Install Node v16
|
|
|
|
become: yes
|
|
|
|
apt:
|
|
|
|
update_cache: yes
|
|
|
|
name: nodejs
|
2021-11-02 18:47:13 -07:00
|
|
|
|
2021-11-02 16:29:52 -07:00
|
|
|
- name: Install Yarn
|
|
|
|
become: yes
|
|
|
|
npm:
|
|
|
|
name: yarn
|
|
|
|
global: yes
|
2021-11-02 18:47:13 -07:00
|
|
|
|
2021-11-03 23:04:25 -07:00
|
|
|
- name: Check for a current app version
|
|
|
|
stat:
|
|
|
|
path: /srv/impress-2020/current
|
|
|
|
register: current_app_version
|
|
|
|
|
|
|
|
- name: Check whether we already have a placeholder app
|
|
|
|
stat:
|
|
|
|
path: /srv/impress-2020/versions/initial-placeholder
|
|
|
|
register: existing_placeholder_app
|
|
|
|
when: not current_app_version.stat.exists
|
|
|
|
|
|
|
|
- name: Create a placeholder app, to run until we deploy a real version
|
|
|
|
command:
|
|
|
|
chdir: /srv/impress-2020/versions
|
|
|
|
cmd: yarn create next-app initial-placeholder
|
|
|
|
when: |
|
|
|
|
not current_app_version.stat.exists and
|
|
|
|
not existing_placeholder_app.stat.exists
|
|
|
|
|
|
|
|
- name: Build the placeholder app
|
|
|
|
command:
|
|
|
|
chdir: /srv/impress-2020/versions/initial-placeholder
|
|
|
|
cmd: yarn build
|
|
|
|
when: not current_app_version.stat.exists
|
|
|
|
|
|
|
|
- name: Set the placeholder app as the current version
|
|
|
|
file:
|
|
|
|
src: /srv/impress-2020/versions/initial-placeholder
|
|
|
|
dest: /srv/impress-2020/current
|
|
|
|
state: link
|
|
|
|
when: not current_app_version.stat.exists
|
|
|
|
|
2021-11-02 22:49:45 -07:00
|
|
|
- name: Install pm2
|
|
|
|
become: yes
|
|
|
|
npm:
|
|
|
|
name: pm2
|
|
|
|
global: yes
|
|
|
|
|
|
|
|
- name: Create pm2 startup script
|
|
|
|
# The current user is going to become the pm2 owner of the app server
|
|
|
|
# process. They'll be able to manage it without `sudo`, including during
|
|
|
|
# normal deploys, and run `pm2 monit` from their shell to see status.
|
|
|
|
become: yes
|
2021-11-13 02:12:05 -08:00
|
|
|
command: "pm2 startup systemd -u {{ ansible_user_id }} --hp /home/{{ ansible_user_id }}"
|
2021-11-02 22:49:45 -07:00
|
|
|
|
|
|
|
- name: Create pm2 ecosystem file
|
|
|
|
copy:
|
2021-11-03 15:43:37 -07:00
|
|
|
content: |
|
2021-11-02 22:49:45 -07:00
|
|
|
module.exports = {
|
|
|
|
apps: [
|
|
|
|
{
|
|
|
|
name: "impress-2020",
|
|
|
|
cwd: "/srv/impress-2020/current",
|
2021-11-03 16:46:35 -07:00
|
|
|
// Instead of `yarn start`, we specify the `next` binary
|
|
|
|
// directly, because it helps pm2 monitor our app correctly.
|
|
|
|
// https://github.com/vercel/next.js/discussions/10675#discussioncomment-34615
|
|
|
|
script: "./node_modules/.bin/next",
|
|
|
|
args: "start --port=3000",
|
2021-11-02 22:49:45 -07:00
|
|
|
instances: "max",
|
|
|
|
exec_mode: "cluster",
|
|
|
|
}
|
|
|
|
]
|
|
|
|
}
|
2021-11-03 15:43:37 -07:00
|
|
|
dest: "~/ecosystem.config.js"
|
|
|
|
# Create a temporary backup file, so we can use it to delete the old
|
|
|
|
# version of the services. (This is important if e.g. a service is
|
|
|
|
# removed or renamed, in which case deleting from the *new* config file
|
|
|
|
# wouldn't include it.)
|
|
|
|
backup: yes
|
|
|
|
register: pm2_ecosystem_file
|
2021-11-02 22:49:45 -07:00
|
|
|
|
2021-11-03 15:43:37 -07:00
|
|
|
- name: Delete old pm2 services if config file changed
|
|
|
|
command: "pm2 delete {{ pm2_ecosystem_file.backup_file | quote }}"
|
|
|
|
when: pm2_ecosystem_file is changed and pm2_ecosystem_file.backup_file is defined
|
|
|
|
|
|
|
|
- name: Delete old pm2 config file if it changed
|
|
|
|
file:
|
|
|
|
path: "{{ pm2_ecosystem_file.backup_file }}"
|
|
|
|
state: absent
|
|
|
|
when: pm2_ecosystem_file is changed and pm2_ecosystem_file.backup_file is defined
|
|
|
|
|
|
|
|
- name: Start pm2 services
|
|
|
|
command: "pm2 start ~/ecosystem.config.js"
|
2021-11-02 22:49:45 -07:00
|
|
|
|
|
|
|
- name: Save pm2 startup script
|
|
|
|
command: pm2 save
|
|
|
|
|
2021-11-03 23:04:25 -07:00
|
|
|
- name: Install nginx
|
|
|
|
become: yes
|
|
|
|
apt:
|
|
|
|
update_cache: yes
|
|
|
|
name: nginx
|
|
|
|
|
2021-11-03 01:00:28 -07:00
|
|
|
- name: Install core snap
|
|
|
|
become: yes
|
|
|
|
community.general.snap:
|
|
|
|
name: core
|
|
|
|
|
|
|
|
- name: Install certbot as a snap
|
|
|
|
become: yes
|
|
|
|
community.general.snap:
|
|
|
|
name: certbot
|
|
|
|
classic: yes
|
|
|
|
|
|
|
|
- name: Set up certbot
|
|
|
|
become: yes
|
|
|
|
command: "certbot certonly --nginx -n --agree-tos --email {{ email_address }} --domains impress-2020-box.openneo.net"
|
|
|
|
|
2021-11-03 00:07:30 -07:00
|
|
|
- name: Add impress-2020 config file to nginx
|
|
|
|
become: yes
|
|
|
|
copy:
|
Add monit watching for nginx and pm2
When I woke up this morning, the app had crashed because the mysql connection was closed!
I'm not sure, why that caused a _crash_? Or why pm2 didn't pick up on it, and said the process was still online? (Maybe the process was running, but the server had stopped?) Those could be good to investigate?…
…but better than diving too far into the details, is to just address the high-level problem: if the app goes down for unexpected reasons, I want it back up!! lol
In this change, we add `monit`, a solid system for monitoring processes (including checking for behavior, like responding to net requests), and configure it to watch the app process and the nginx process.
To test, you can run `pm2 stop impress-2020`, or `systemctl stop nginx`, to see that Monit brings them back up within seconds!
This does add some potential surprise if you're _trying_ to take the processes down. The easiest way is to send the stop command through monit, like `monit stop nginx`. This will disable monitoring until you start it again through monit, I think? (You can also disable/enable monitoring as a direct command, regardless of app state.)
2021-11-03 16:32:14 -07:00
|
|
|
content: |
|
2021-11-03 00:07:30 -07:00
|
|
|
server {
|
2021-11-03 01:00:28 -07:00
|
|
|
server_name impress-2020-box.openneo.net;
|
2021-11-03 00:07:30 -07:00
|
|
|
listen 80;
|
2024-02-13 08:43:06 -08:00
|
|
|
listen [::]:80;
|
2021-11-03 01:00:28 -07:00
|
|
|
if ($host = impress-2020-box.openneo.net) {
|
|
|
|
return 301 https://$host$request_uri;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
server {
|
2021-11-03 00:07:30 -07:00
|
|
|
server_name impress-2020-box.openneo.net;
|
2021-11-03 01:00:28 -07:00
|
|
|
listen 443 ssl;
|
2024-02-13 08:43:06 -08:00
|
|
|
listen [::]:443 ssl;
|
2021-11-03 01:00:28 -07:00
|
|
|
ssl_certificate /etc/letsencrypt/live/impress-2020-box.openneo.net/fullchain.pem;
|
|
|
|
ssl_certificate_key /etc/letsencrypt/live/impress-2020-box.openneo.net/privkey.pem;
|
|
|
|
include /etc/letsencrypt/options-ssl-nginx.conf;
|
|
|
|
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
|
|
|
|
ssl_session_cache shared:SSL:10m; # https://superuser.com/q/1484466/14127
|
|
|
|
|
2021-11-03 00:07:30 -07:00
|
|
|
# TODO: Serve static files directly, instead of through the proxy
|
|
|
|
location / {
|
|
|
|
proxy_pass http://127.0.0.1:3000;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
dest: /etc/nginx/sites-enabled/impress-2020
|
|
|
|
notify:
|
|
|
|
- Restart nginx
|
|
|
|
|
2022-10-11 12:54:02 -07:00
|
|
|
- name: Create cron log files directory
|
|
|
|
file:
|
|
|
|
path: /srv/impress-2020/cron-logs
|
|
|
|
state: directory
|
|
|
|
|
2023-10-15 14:43:41 -07:00
|
|
|
- name: Set up modeling cron jobs
|
2022-10-11 12:54:02 -07:00
|
|
|
cron:
|
|
|
|
name: model-needed-items
|
|
|
|
minute: "*/10" # Every 10 minutes
|
|
|
|
job: "cd /srv/impress-2020/current && yarn model-needed-items 2>&1 > /srv/impress-2020/cron-logs/model-needed-items.log"
|
|
|
|
|
2023-10-15 14:43:41 -07:00
|
|
|
# Idk why the app has been so memory leaky lately, but let's just restart it
|
|
|
|
# like *constantly* tbh.
|
|
|
|
- name: Set up cron job to restart the app
|
|
|
|
cron:
|
|
|
|
name: restart-impress-2020
|
|
|
|
minute: "*/8" # Every 8 minutes
|
|
|
|
job: "pm2 restart impress-2020"
|
|
|
|
|
2021-11-02 22:49:45 -07:00
|
|
|
- name: Install dependencies for the npm module node-canvas
|
2021-11-02 16:29:52 -07:00
|
|
|
become: yes
|
|
|
|
apt:
|
|
|
|
update_cache: yes
|
|
|
|
name:
|
|
|
|
- build-essential
|
|
|
|
- libcairo2-dev
|
|
|
|
- libpango1.0-dev
|
|
|
|
- libjpeg-dev
|
|
|
|
- libgif-dev
|
|
|
|
- librsvg2-dev
|
2021-11-12 21:20:48 -08:00
|
|
|
- name: Install Playwright system dependencies
|
|
|
|
# NOTE: I copied the package list from the source list for
|
|
|
|
# `npx playwright install-deps`, which I couldn't get running in
|
|
|
|
# Ansible as root, and besides, I prefer manually managing the
|
|
|
|
# package list over running an npm script as root!
|
2021-11-13 02:16:58 -08:00
|
|
|
# TODO: We're using Puppeteer now, should this list change in some way?
|
2021-11-12 21:20:48 -08:00
|
|
|
become: yes
|
|
|
|
apt:
|
|
|
|
update_cache: yes
|
|
|
|
name:
|
|
|
|
# Tools
|
|
|
|
- xvfb
|
|
|
|
- fonts-noto-color-emoji
|
|
|
|
- ttf-unifont
|
|
|
|
- libfontconfig
|
|
|
|
- libfreetype6
|
|
|
|
- xfonts-cyrillic
|
|
|
|
- xfonts-scalable
|
|
|
|
- fonts-liberation
|
|
|
|
- fonts-ipafont-gothic
|
|
|
|
- fonts-wqy-zenhei
|
|
|
|
- fonts-tlwg-loma-otf
|
|
|
|
- ttf-ubuntu-font-family
|
|
|
|
# Chromium
|
|
|
|
- fonts-liberation
|
|
|
|
- libasound2
|
|
|
|
- libatk-bridge2.0-0
|
|
|
|
- libatk1.0-0
|
|
|
|
- libatspi2.0-0
|
|
|
|
- libcairo2
|
|
|
|
- libcups2
|
|
|
|
- libdbus-1-3
|
|
|
|
- libdrm2
|
|
|
|
- libegl1
|
|
|
|
- libgbm1
|
|
|
|
- libglib2.0-0
|
|
|
|
- libgtk-3-0
|
|
|
|
- libnspr4
|
|
|
|
- libnss3
|
|
|
|
- libpango-1.0-0
|
|
|
|
- libx11-6
|
|
|
|
- libx11-xcb1
|
|
|
|
- libxcb1
|
|
|
|
- libxcomposite1
|
|
|
|
- libxdamage1
|
|
|
|
- libxext6
|
|
|
|
- libxfixes3
|
|
|
|
- libxrandr2
|
|
|
|
- libxshmfence1
|
2021-11-13 02:16:58 -08:00
|
|
|
- name: Enable user namespace cloning for Chromium sandboxing
|
|
|
|
become: yes
|
|
|
|
ansible.posix.sysctl:
|
|
|
|
name: kernel.unprivileged_userns_clone
|
|
|
|
value: "1"
|
|
|
|
|
2021-11-03 00:07:30 -07:00
|
|
|
handlers:
|
|
|
|
- name: Restart nginx
|
|
|
|
become: yes
|
Add monit watching for nginx and pm2
When I woke up this morning, the app had crashed because the mysql connection was closed!
I'm not sure, why that caused a _crash_? Or why pm2 didn't pick up on it, and said the process was still online? (Maybe the process was running, but the server had stopped?) Those could be good to investigate?…
…but better than diving too far into the details, is to just address the high-level problem: if the app goes down for unexpected reasons, I want it back up!! lol
In this change, we add `monit`, a solid system for monitoring processes (including checking for behavior, like responding to net requests), and configure it to watch the app process and the nginx process.
To test, you can run `pm2 stop impress-2020`, or `systemctl stop nginx`, to see that Monit brings them back up within seconds!
This does add some potential surprise if you're _trying_ to take the processes down. The easiest way is to send the stop command through monit, like `monit stop nginx`. This will disable monitoring until you start it again through monit, I think? (You can also disable/enable monitoring as a direct command, regardless of app state.)
2021-11-03 16:32:14 -07:00
|
|
|
systemd:
|
2021-11-03 00:07:30 -07:00
|
|
|
name: nginx
|
|
|
|
state: restarted
|