Update README, improvements

README.md

@@ -3,8 +3,7 @@
 ```
 .
 |-- Makefile         # Wrapper to simplify interaction
-|-- config.json      # Read by Make, Terraform, Packer
-|-- deploymentagent
+|-- deploymentagent  # Can be used to trigger deployments manually or from a pipeline
 |-- infrastructure   # Terraform modules
 |   |-- compute      # Loads the compute module
 |   |-- environment  # Loads the environment module and provides outputs
@@ -12,17 +11,16 @@
 |   |-- storage      # Loads the storage module
 |   `-- modules      # Contains the code for all the modules
 |-- nixos            # NixOS image builder with Packer
+|-- config.json      # Read by Make, Terraform, Packer
 |-- secrets.json     # Read by Make, Terraform, Packer
 `-- vault            # Policy examples
 ```
 
 ## Overview
-
 ### Tools and Dependencies
 - Terraform 0.13.x [https://releases.hashicorp.com/terraform/](https://releases.hashicorp.com/terraform/)
 - Packer 1.6.x [https://releases.hashicorp.com/packer/](https://releases.hashicorp.com/packer/)
 - Make & Unix command line tools
-Optional:
 - Vault CLI 1.6.x [https://releases.hashicorp.com/vault/](https://releases.hashicorp.com/vault/)
 
 ### Configuration
@@ -65,9 +63,7 @@ git secret reveal
 #### Gitlab
 Secrets, such as the SSH key pair for the default system user are stored in the [Gitlab CI/CD](https://gitlab.com/infektweb/glv5/hetzner-cloud-environment/-/settings/ci_cd) settings page of this Git project (for now), in the *Variables* section.
 
-https://gitlab.com/infektweb/glv5/hetzner-cloud-environment/-/settings/repository/#js-deploy-tokens
-
-id\_rsa\_operator_pub is baked into the image generated by Packer (see `nixos/nix/system.nix`)
+'id\_rsa\_operator_pub' is baked into the image generated by Packer (see `nixos/nix/system.nix`).
 
 ### NixOS
 #### Building NixOS Images (Snapshots) with Packer
@@ -146,13 +142,55 @@ $ make infra-destroy MODULE=compute
 ```
 
 ## Operations Guide
+### Components
+![diagram](./docs/diagram.png)
+(the file `docs/diagram.png` can be edited with [https://app.diagrams.net/](https://app.diagrams.net/)
+
+### Sytemd Units
+The different services can be found via their name in `systemctl`.
+Containers managed by Podman are prefixed with `podman-`, for example: `podman-api`, `podman-web` etc.
+
+#### Rebuild NixOS via Systemd
+There is a special unit called 'nixos-rebuild' with which `nixos-rebuild switch` can be called in the background, either manually, periodically or based on an event, such as via socket activation:
+```sh
+$ nc -v localhost 4444
+```
+The connection will be kept open and closed by Systemd as soon as the rebuild and switch have been completed (switch only in case the build has been successful).
 
 ### Data
 #### Ephemeral Data
-- /opt/
-- /etc/nixos
+In case the server is deleted, these files will be lost and need to be re-supplied via Cloud-Init again upon server creation.
+###### /opt/
+Data in `/opt` is supplied by Cloud-Init and should not be modified.
+The files will be read by different components such as Nix as well as the Certbot wrapper, which is located at `/opt/certbot.sh`.
+```
+- /opt/certbot.sh
+- /opt/cloud-init-misc-data/
+  - aws_access_key_id # used by Certbot to manage DNS records
+  - aws_secret_access_key # used by Certbot to manage DNS records
+  - domain_alternative_names # used by Certbot to create certificate
+  - domain_name # used by Certbot to create certificate
+  - hcloud_token # used by Certbot to manage load balancers
+  - elasticsearch_password # default password for users 'elastic' and 'kibana'
+  - environment # name of the environment which is consumed by guidelines.nix
+  - guidelines.json # default launch specification (see `./deploymentagent/`)
+  - vault_db_password # default password for Postgres Vault storage backend
+```
+###### /etc/nixos
+`/etc/nixos` contains all the Nix files part of the base image created with Packer as well as the files supplied via Cloud-Init.
 #### Persistent Data
+An external volume provided by Hetzner Cloud is mounted Data which is stored in `/mnt/data`0
 - /mnt/data
+```
+- guidelines.json # launch specification (see `./deploymentagent/`)
+- elasticsearch # Elasticsearch home
+- kibana # Kibana home
+- letsencrypt # Let's Encrypt data (configuration, logs, ..) generated and used by Certbot
+- postgresql # PostgreSQL data directory
+- vault-deploymentagent-token
+- vault-guidelines-api-token
+- vault-master-key # optional: Can be added to automatically unseal Vault
+```
 
 ### Setting Up a New Environment
 The following sections assume the environment to be called 'production'.
@@ -167,7 +205,7 @@ Set the environment name, domain names and desired NixOS image/snapshot ID in `c
     "domain_alternative_names_production": "*.guidelines.ch"
 }
 ```
-Use your personal Gitlab deployment- and Hetzner Cloud tokens.
+It's best to use your personal Gitlab deployment token and a generic Hetzner Cloud token, since the latter will also be used by the created server(s).
 `secrets.json`:
 ```json
 {
@@ -219,7 +257,7 @@ $ sudo -i
 ```
 
 #### Configuring Certbot
-In case you have an existing configuration for Certbot, you can simply copy it to `/mnt/data/letsencrypt`, otherwise you can set up a new configuration either locally, or directly on the server itself.
+In case you have an existing configuration for Certbot, you can copy it to `/mnt/data/letsencrypt`, otherwise you can set up a new configuration either locally, or directly on the server itself:
 ```sh
 $ export AWS_ACCESS_KEY_ID="..."
 $ export AWS_SECRET_ACCESS_KEY="..."
@@ -237,37 +275,65 @@ $ systemctl start nixos-rebuild
 $ systemctl start hetzner-certbot
 $ journalctl -u hetzner-certbot
 ```
+Keep in mind that when running the `hetzner-certbot` unit, all services exposed by the load balancer will be briefly "offline" (~ 5-10 seconds) as due to the before mentioned limitations, the certificate references will need to be deleted and recreated by `certbot.sh`.
 
 #### Configuring Vault
-You can access Vault on port 9443 via any hostname behind the load balancer [https://guidelines.ch:9443/](https://guidelines.ch:9443/).
+##### Accessing Vault
+To access the Vault API and web UI from your local machine, you will need to configure a port forwarding via SSH.
+```sh
+$ while :; do ssh -L 4443:10.0.1.51:8200 -N <server ip>; done
+```
+The vault web UI can then be accessed via [http://localhost:4443/ui/](http://localhost:4443/ui/)
+##### Initial Setup of Vault
 As a first step, you will need to create a master key (set) which is used to unseal Vault on each startup.
 To use just one master key, initialize Vault with "Key shares" and "Key threshold" both set to "1".
 The "initial root token" is used to authenticate as an administrator with the Vault API or web UI.
 The "key" is used to unseal Vault in case it has been sealed (manually or due to a restart).
-You can now set up the key-value based secret engine which is supported by the [settings](https://gitlab.com/infektcommon/settings) package.
-Be sure to use V2 of the KV engine.
+You can now set up the key-value based secrets engine which is supported by the [settings](https://gitlab.com/infektcommon/settings) package (only V2 is supported).
 See the [Vault documentation](https://www.vaultproject.io/docs).
 
+##### Unsealing Vault
 To unseal Vault manually, you can either use curl, the Vault CLI, or use the prompt on the web UI.
 ```sh
-$ curl -XPUT http://127.0.0.1:8200/v1/sys/unseal -d '{"key": "master key"}'
+$ curl -XPUT http://localhost:4443/v1/sys/unseal -d '{"key": "master key"}'
 ```
 ```sh
-$ vault operator unseal
+$ vault operator unseal -address=http://localhost:4443
 Key (will be hidden):
 ```
+Note that 'v1' of the Vault API is unrelated to 'v2' of the secret engines.
 
-##### Unseal Vault Automatically on Startup
-You can manually write the created master key to `/mnt/data/vault-root-token`.
+##### Unsealing Vault Automatically
+You can manually write the created master key to `/mnt/data/vault-master-key`.
 If this file exists and contains a valid master key, Vault will be unsealed automatically on startup.
 
+##### Example Role
+```hcl
+path "kv/data/guidelines/production/api" {
+  capabilities = ["read"]
+}
+```
+'kv' relates to the name of the secrets engine created.
+The path segment '/data/' must always be placed between the name of the secret engine and secret path when using V2 of the engine APIs.
+
+##### Creating Vault Tokens for Services
+Refer to the [Vault documentation](https://www.vaultproject.io/docs/commands/token/create) to see how to manually create a token with a policy.
+To take it a step further and generate Vault tokens upon container start (with some help of additional tooling) refer to the [AppRole Authentication Method](https://www.vaultproject.io/docs/auth/approle)
+
 #### Configuring Elasticsearch
-Kibana can be accessed on port 8443 via any hostname behind the load balancer [https://guidelines.ch:8443/](https://guidelines.ch:9443/).
-The password is derieved from the `elasticsearch_password_<env>` key in `secrets.json`
+The 'elastic' admin user as well as the user for Kibana are derieved from the `elasticsearch_password_<env>` key in `secrets.json`
+Kibana can be accessed on port 8443 via any hostname behind the load balancer [https://guidelines.ch:8443/](https://guidelines.ch:8443/).
+
+#### Deployment Agent
+The deployment agent can be accessed on port 9443 via any hostname behind the load balancer [https://guidelines.ch:9443/](https://guidelines.ch:9443/).
+See `./deploymentagent` for how to work with it.
+In a new environment, every container of the Guidelines service will be provisioned based on the 'latest' tag.
 
 #### Configuring Guidelines
-```
+Create a database
+```sql
 CREATE DATABASE guidelines;
 CREATE USER guidelines WITH ENCRYPTED PASSWORD 'changeme';
 GRANT ALL PRIVILEGES ON DATABASE guidelines TO guidelines;
 ```
+For the time being, until Vault support has been rolled out to all services, change the file `./infrastructure/modules/compute/nix/guidelines.nix` locally before deploying a server with Terraform or manually edit `/etc/nixos/guidelines.nix` on the server directly and rebuild with `systemctl start nixos-rebuild` to get your services up and running for the testing environment

infrastructure/modules/compute/cloudinit.tpl

@@ -77,7 +77,7 @@ write_files:
       }
     owner: root:root
     permissions: '0600'
-  - path: /opt/guidelines.json
+  - path: /opt/cloud-init-misc-data/guidelines.json
     content: |
       {
         "api": "latest",

infrastructure/modules/compute/nix/guidelines.nix

@@ -23,16 +23,14 @@ in
       ];
       environment = {
         "PORT" = "8080";
-        "BASE_CLIENT_URL" = "http://[space].test-glv5.guidelines.ch";
+        "BASE_CLIENT_URL" = "http://[space].guidelines.ch";
         "ENVIRONMENT" = (builtins.readFile /opt/cloud-init-misc-data/environment);
         "VAULT_SECRET_PATH" = "kv/data/guidelines/${(builtins.readFile /opt/cloud-init-misc-data/environment)}/api";
         "VAULT_URL" = "http://host:8200";
-        
       };
       volumes = [
         "/mnt/data/vault-guidelines-api-token:/vault-token"
       ];
-      #extraDockerOptions = [ "--network=foo" ];
     };
 
     oci-containers.containers."web" = {
@@ -45,6 +43,17 @@ in
       ];
       environment = {
         "API_URL" = "http://host:8001";
+        "PORT" = "8080";
+        "CLIENT_URL" = "https://guidelines.ch";
+        "REDIS_URL" = "redis://host:6379";
+
+        "AUTH0_AUDIENCE" = "";
+        "AUTH0_CLIENT_ID" = "";
+        "AUTH0_CLIENT_SECRET" = "";
+        "AUTH0_DOMAIN" = "";
+        "CHALLENGE_CONTENT" = "";
+        "CHALLENGE_ID" = "";
+
       };
     };
 
@@ -63,21 +72,18 @@ in
         "DEPLOYMENT_STATE_FILE" = "/guidelines.json";
       };
       volumes = [
-        "/mnt/data/vault-deploymentagent-api-token:/vault-token"
+        "/mnt/data/vault-deploymentagent-token:/vault-token"
         "/run/podman-containers.sock:/tmp/podman/podman.sock"
         "/mnt/data/guidelines.json:/guidelines.json"
       ];
-      #extraDockerOptions = [ "--network=foo" ];
     };
 
-    #oci-containers.containers."containerapi" = {
-    #  image = "alpine";
-    #  volumes = [
-    #    "/run/podman-containers.sock:/podman-containers.sock"
-    #  ];
-    #  entrypoint = "/bin/sleep";
-    #  cmd = ["10000"];
-    #};
+    # oci-containers.containers."html2pdf" = {
+    # };
+
+    # oci-containers.containers."filestore" = {
+    # };
+
   };
 
   systemd.services.docker-podman-rest-api = {
@@ -89,8 +95,26 @@ in
     '';
   };
 
-  services.redis.enable = true;
-  services.redis.requirePass = "p15c4e6538de2061edd65a52ab216ba071d78b1532a937c1c3d5821d5c571c0cf";
+  system.activationScripts = {
+    guidelinesJson = {
+      text = "test -f /mnt/data/guidelines.json || cp /opt/cloud-init-misc-data/guidelines.json /mnt/data/guidelines.json";
+      deps = [];
+    };
+  };
+
+  services.redis = {
+    enable = true;
+    bind = "10.0.1.51";
+    vmOverCommit = true;
+    extraConfig = ''
+      protected-mode no
+    '';
+  };
+  systemd.services.redis.serviceConfig = {
+    ReadWriteDirectories = "/var/lib/redis";
+    TimeoutStartSec = "60";
+    TimeoutStopSec = "60";
+  };
 
   networking.firewall.allowedTCPPorts = [ 6379 5000 4444 ];
 }

infrastructure/modules/compute/nix/vault.nix

@@ -18,7 +18,7 @@
     wantedBy = [ "multi-user.target" ];
     after = [ "vault.service" ];
     script = ''
-           file=/mnt/data/vault-root-token; test -f "$file" || { printf "Stopping automatic unseal, no token present at $file\n"; exit; }; count=0; while [ "$count" -le 10 ]; do count=`expr "$count" + 1`; printf "=> Trying to unseal Vault..\n"; /run/current-system/sw/bin/curl -XPUT http://127.0.0.1:8200/v1/sys/unseal -d '{"key": "'$(head -n 1 $file)'"}' && break; sleep 10; done
+           file=/mnt/data/vault-master-key; test -f "$file" || { printf "Stopping automatic unseal, no key present at $file\n"; exit; }; count=0; while [ "$count" -le 10 ]; do count=`expr "$count" + 1`; printf "=> Trying to unseal Vault..\n"; /run/current-system/sw/bin/curl -XPUT http://127.0.0.1:8200/v1/sys/unseal -d '{"key": "'$(head -n 1 $file)'"}' && break; sleep 10; done
     '';
   };
 

vault/example_service_policy.hcl

@@ -1,8 +0,0 @@
-# Read-only access to all secrets in key-value store
-path "kv/data/guidelines/production/api" {
-  capabilities = ["read"]
-}
-
-path "kv/data/guidelines/staging/api" {
-  capabilities = ["read"]
-}