From deabdd17c6a60cefeb2d3f81e1cb6409ebb2d25a Mon Sep 17 00:00:00 2001 From: Akash Askoolum Date: Thu, 23 Apr 2020 10:33:00 +0100 Subject: [PATCH] revise layout of docs Taking inspiration from the frontend repo, provide more structure to docs and generate a TOC. --- README.md | 4 +- VISION.md => docs/00-about/01-vision.md | 0 docs/{ => 00-about}/images/roundel.jpg | Bin .../images/screenshot-2015-07-03T11:34:43.jpg | Bin docs/01-setup/01-software-dependencies.md | 23 ++++++ docs/01-setup/02-aws-dependencies.md | 28 +++++++ docs/01-setup/03-nginx.md | 10 +++ docs/01-setup/04-configuration.md | 7 ++ docs/02-running/01-running-locally.md | 34 +++++++++ docs/02-running/02-kahuna.md | 11 +++ docs/02-running/03-logging.md | 19 +++++ .../01-authentication.md} | 0 docs/04-troubleshooting/01-nginx.md | 19 +++++ docs/04-troubleshooting/02-sbt.md | 6 ++ docs/{ => 99-archives}/01-cloudformation.md | 0 docs/{ => 99-archives}/02.01-dev-setup.md | 0 docs/{ => 99-archives}/02.02-configuration.md | 0 docs/{ => 99-archives}/03-running.md | 0 docs/{ => 99-archives}/04.01-api.md | 0 docs/99-archives/04.02-authentication.md | 47 ++++++++++++ docs/{ => 99-archives}/04.03-upload-image.md | 0 docs/{ => 99-archives}/04.04-start-thrall.md | 0 docs/{ => 99-archives}/04.05-media-api.md | 0 docs/{ => 99-archives}/05-logging.md | 0 docs/99-archives/README.md | 21 ++++++ .../99-archives/TROUBLESHOOTING.md | 0 docs/{ => 99-archives}/elasticsearch6.md | 0 docs/{ => 99-archives}/elasticsearch7.md | 0 docs/{ => 99-archives}/local-kinesis.md | 0 docs/README.md | 70 +++++++++++++----- docs/generate-toc.sh | 62 ++++++++++++++++ docs/how-to-create-a-doc-file.md | 8 ++ 32 files changed, 347 insertions(+), 22 deletions(-) rename VISION.md => docs/00-about/01-vision.md (100%) rename docs/{ => 00-about}/images/roundel.jpg (100%) rename docs/{ => 00-about}/images/screenshot-2015-07-03T11:34:43.jpg (100%) create mode 100644 docs/01-setup/01-software-dependencies.md create mode 100644 docs/01-setup/02-aws-dependencies.md create mode 100644 docs/01-setup/03-nginx.md create mode 100644 docs/01-setup/04-configuration.md create mode 100644 docs/02-running/01-running-locally.md create mode 100644 docs/02-running/02-kahuna.md create mode 100644 docs/02-running/03-logging.md rename docs/{04.02-authentication.md => 03-api-access/01-authentication.md} (100%) create mode 100644 docs/04-troubleshooting/01-nginx.md create mode 100644 docs/04-troubleshooting/02-sbt.md rename docs/{ => 99-archives}/01-cloudformation.md (100%) rename docs/{ => 99-archives}/02.01-dev-setup.md (100%) rename docs/{ => 99-archives}/02.02-configuration.md (100%) rename docs/{ => 99-archives}/03-running.md (100%) rename docs/{ => 99-archives}/04.01-api.md (100%) create mode 100644 docs/99-archives/04.02-authentication.md rename docs/{ => 99-archives}/04.03-upload-image.md (100%) rename docs/{ => 99-archives}/04.04-start-thrall.md (100%) rename docs/{ => 99-archives}/04.05-media-api.md (100%) rename docs/{ => 99-archives}/05-logging.md (100%) create mode 100644 docs/99-archives/README.md rename TROUBLESHOOTING.md => docs/99-archives/TROUBLESHOOTING.md (100%) rename docs/{ => 99-archives}/elasticsearch6.md (100%) rename docs/{ => 99-archives}/elasticsearch7.md (100%) rename docs/{ => 99-archives}/local-kinesis.md (100%) create mode 100755 docs/generate-toc.sh create mode 100644 docs/how-to-create-a-doc-file.md diff --git a/README.md b/README.md index 854ecae7a3..c8508f168b 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ management system**, which provides a **universal** and **fast** experience accessing media that is **organised** and using it in an **affordable** way to produce **high-quality** content. -See the [Vision](VISION.md) document for more details on the core +See the [Vision](docs/00-about/01-vision.md) document for more details on the core principles behind this project. -![Screenshot of Grid search](docs/images/screenshot-2015-07-03T11:34:43.jpg) +![Screenshot of Grid search](docs/00-about/images/screenshot-2015-07-03T11:34:43.jpg) Grid runs as a set of independent micro-services ([Scala](http://www.scala-lang.org/) and diff --git a/VISION.md b/docs/00-about/01-vision.md similarity index 100% rename from VISION.md rename to docs/00-about/01-vision.md diff --git a/docs/images/roundel.jpg b/docs/00-about/images/roundel.jpg similarity index 100% rename from docs/images/roundel.jpg rename to docs/00-about/images/roundel.jpg diff --git a/docs/images/screenshot-2015-07-03T11:34:43.jpg b/docs/00-about/images/screenshot-2015-07-03T11:34:43.jpg similarity index 100% rename from docs/images/screenshot-2015-07-03T11:34:43.jpg rename to docs/00-about/images/screenshot-2015-07-03T11:34:43.jpg diff --git a/docs/01-setup/01-software-dependencies.md b/docs/01-setup/01-software-dependencies.md new file mode 100644 index 0000000000..3392a4cfc3 --- /dev/null +++ b/docs/01-setup/01-software-dependencies.md @@ -0,0 +1,23 @@ +# Software dependencies + +Grid requires the following to be installed: +- sbt +- Java 8 +- node +- nginx +- Docker +- [GD](http://libgd.github.io/) +- GraphicsMagick with `little-cms2` +- ImageMagick +- pngquant +- [awscli](https://aws.amazon.com/cli/) +- [jq](https://stedolan.github.io/jq/) +- [exiftool](http://www.sno.phy.queensu.ca/~phil/exiftool/) +- [md5Sum](http://www.ahwkong.com/post/2011/06/07/p-6255384955/) +- [dev-nginx](https://github.com/guardian/dev-nginx) + +If you're a [homebrew](https://brew.sh/) user, you can use the [`Brewfile`](../../Brewfile) in the project root: + +```shell script +brew bundle +``` diff --git a/docs/01-setup/02-aws-dependencies.md b/docs/01-setup/02-aws-dependencies.md new file mode 100644 index 0000000000..657137ba3d --- /dev/null +++ b/docs/01-setup/02-aws-dependencies.md @@ -0,0 +1,28 @@ +# AWS dependencies + +Grid requires various AWS resources to run, such as S3 buckets and DynamoDB tables. + +## Credentials +Grid interacts with various AWS resources with the +[profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-multiple-profiles.html) `media-service`. + +Create a profile using the AWS CLI: + +```shell script +aws configure --profile media-service +``` + +Developers working at the Guardian can use Janus to get credentials. + +## Resources +The resources needed to run Grid locally are defines in the CloudFormation template [here](../../cloud-formation/dev-template.yaml). + +Use this template to create a CloudFormation stack; for the purposes of this documentation we'll assume a stack name of `media-service-DEV`. + +```shell script +aws cloudformation create-stack \ + --stack-name media-service-DEV \ + --template-body file://cloud-formation/dev-template.yaml \ + --profile media-service \ + --region eu-west-1 +``` diff --git a/docs/01-setup/03-nginx.md b/docs/01-setup/03-nginx.md new file mode 100644 index 0000000000..f9d497808e --- /dev/null +++ b/docs/01-setup/03-nginx.md @@ -0,0 +1,10 @@ +# NGINX + +Grid consists of a number of micro services. Locally, we run each behind an NGINX proxy. + +These proxies can be configured using the [`dev-nginx`](https://github.com/guardian/dev-nginx) tool, +and the configuration file [`nginx-mappings.yml`](../../nginx-mappings.yml): + +```shell script +dev-nginx setup-app nginx-mappings.yml +``` diff --git a/docs/01-setup/04-configuration.md b/docs/01-setup/04-configuration.md new file mode 100644 index 0000000000..6a5408d2d9 --- /dev/null +++ b/docs/01-setup/04-configuration.md @@ -0,0 +1,7 @@ +# Configuration + +Grid sources its configuration from `/etc/gu`. + +We can use the [`generate-dot-properties`](../../scripts/generate-dot-properties) scripts to generate these. + +Developers working at the Guardian can use the [`fetch-config.sh`](../../fetch-config.sh) script to download pre-generated files. diff --git a/docs/02-running/01-running-locally.md b/docs/02-running/01-running-locally.md new file mode 100644 index 0000000000..c3465edf3d --- /dev/null +++ b/docs/02-running/01-running-locally.md @@ -0,0 +1,34 @@ +# Running locally + +By now you should have: +- installed all software dependencies +- created an AWS profile called `media-service` +- created an AWS CloudFormation stack +- setup nginx proxies +- generated configuration files in `/etc/gu` + +We can start Grid by using the [dev-start.sh](../../dev-start.sh) script: + +```shell script +./dev-start.sh +``` + +The following flags are available for `dev-start.sh`: +- `--debug` which will make port `5005` available for [debugging in IntelliJ](https://www.jetbrains.com/help/idea/attaching-to-local-process.html) +- `--ship-logs` which will ship logs to a [local elk](https://github.com/guardian/local-elk) stack over tcp on port `5000` + +## `the_pingler.sh` +Grid consists of many micro-services. Some services don't compile until you hit them for the first time. +There is a script included that will do this for you called [the_pingler.sh](../the_pingler.sh). +{ +This is a simple shell script that keeps pinging the healthcheck endpoints of the various +services and reports it via the colour of the URL. This is needed because some services do +not start to function correctly until they have been contacted at least once. +It's recommended to keep this running in the background while you have the stack started up +and keep it running in a background terminal. + +```bash +./the_pingler.sh +``` + +If everything has worked, we should be able to access Grid in a browser on `https://media.local.dev-gutools.co.uk`. diff --git a/docs/02-running/02-kahuna.md b/docs/02-running/02-kahuna.md new file mode 100644 index 0000000000..04dad3215a --- /dev/null +++ b/docs/02-running/02-kahuna.md @@ -0,0 +1,11 @@ +# Kahuna + +Kahuna is the name of Grid's client-side web app. + +Grid uses webpack to bundle client side code. + +Run the following from the `kahuna` directory to watch for changes: + +```shell script +npm run watch +``` diff --git a/docs/02-running/03-logging.md b/docs/02-running/03-logging.md new file mode 100644 index 0000000000..bd808c12de --- /dev/null +++ b/docs/02-running/03-logging.md @@ -0,0 +1,19 @@ +# Accessing logs + +Grid logs quite heavily to provide an insight into it's usage, behaviour and health. + +By default, each micro-service logs to their relative `logs` directory; `kahuna`'s logs are in `./kahuna/logs/`. + +In PROD, we push logs to an [ELK stack](https://www.elastic.co/what-is/elk-stack) using the [kinesis-logback-appender](https://github.com/guardian/kinesis-logback-appender). +An ELK stack provides a very simple and easy to use way of interrogating logs. + +We can run an ELK stack locally using [local-elk](https://github.com/guardian/local-elk). +Once we have local-elk running, we can ship logs to it using the `--ship-logs` flag on the `dev-start.sh` script: + +```shell script +./dev-start --ship-logs +``` + +## Stopwatch +Wrap any function `do_thing` with a call to Stopwatch: `Stopwatch("thing") {do_thing}` to get +a log line on success giving the duration in ns. diff --git a/docs/04.02-authentication.md b/docs/03-api-access/01-authentication.md similarity index 100% rename from docs/04.02-authentication.md rename to docs/03-api-access/01-authentication.md diff --git a/docs/04-troubleshooting/01-nginx.md b/docs/04-troubleshooting/01-nginx.md new file mode 100644 index 0000000000..d04ae52bd2 --- /dev/null +++ b/docs/04-troubleshooting/01-nginx.md @@ -0,0 +1,19 @@ +# NGINX + +## NGINX returns "413 Request Entity Too Large" +Make sure you bump the maximum allowed body size in your nginx config (defaults to 1MB): + +``` +client_max_body_size 20m; +``` + +## NGINX, Play & SNI +As the Play Framework does not yet support [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication) + NGINX can't always work out which certificate to send where when there are multiple services on the same IP. + This might result in NGINX sending the incorrect certificate resulting in a `HostnameVerifier Exception`. + +## Resolution +When the correct cert to send is ambiguous, NGINX simply sends the first cert it sees in it's configuration, +which is loaded from config files in alphabetical order. + +To resolve this problem, prefix your grid config filename with `0-`. diff --git a/docs/04-troubleshooting/02-sbt.md b/docs/04-troubleshooting/02-sbt.md new file mode 100644 index 0000000000..0cb2589110 --- /dev/null +++ b/docs/04-troubleshooting/02-sbt.md @@ -0,0 +1,6 @@ +# SBT + +## Compilation fails because dependencies that should exist cannot be found +- Kill all java process, then run `sbt clean` and `sbt clean-files` +- If this doesn't help, try removing all target files in the project and recompile +- If this still doesn't work, try cleaning ivy cache in ~/.ivy2.cache diff --git a/docs/01-cloudformation.md b/docs/99-archives/01-cloudformation.md similarity index 100% rename from docs/01-cloudformation.md rename to docs/99-archives/01-cloudformation.md diff --git a/docs/02.01-dev-setup.md b/docs/99-archives/02.01-dev-setup.md similarity index 100% rename from docs/02.01-dev-setup.md rename to docs/99-archives/02.01-dev-setup.md diff --git a/docs/02.02-configuration.md b/docs/99-archives/02.02-configuration.md similarity index 100% rename from docs/02.02-configuration.md rename to docs/99-archives/02.02-configuration.md diff --git a/docs/03-running.md b/docs/99-archives/03-running.md similarity index 100% rename from docs/03-running.md rename to docs/99-archives/03-running.md diff --git a/docs/04.01-api.md b/docs/99-archives/04.01-api.md similarity index 100% rename from docs/04.01-api.md rename to docs/99-archives/04.01-api.md diff --git a/docs/99-archives/04.02-authentication.md b/docs/99-archives/04.02-authentication.md new file mode 100644 index 0000000000..4c3792807f --- /dev/null +++ b/docs/99-archives/04.02-authentication.md @@ -0,0 +1,47 @@ +# Authentication + +There are two ways to authenticate with Grid. + +## Panda Auth +This is typically used for client-server authentication. + +We use the [pan-domain-authentication](https://github.com/guardian/pan-domain-authentication) library to authenticate +using Google. + +## API Keys +This is typically used for server-server authentication. + +API keys are stored in the `KeyBucket` of the cloudformation stack; drop a key file here and it'll get picked by Grid +within 10 minutes and you'll be able to make authenticated calls. + +### Creating a Key +Generate a key file with a random string as the filename. The contents of the file should be the application name. + +```bash +export APP=test && echo $APP > $APP-`head -c 1024 /dev/urandom | md5 | tr '[:upper:]' '[:lower:]' | cut -c1-20` +``` + +Now you have a key file, find your `KeyBucket`: + +```bash +./get-stack-resource.sh KeyBucket +``` + +Then copy your key: + +```bash +aws s3 cp /path/to/file s3://bucket --profile media-service +``` + +Be sure to delete the key from your local machine! + +### Testing a Key +Once you've copied your key up to the `KeyBucket` it'll be picked up by the apps within 10 minutes. + +You can test your key by making a curl request: + +```bash +curl -s -I -X GET -H "X-Gu-Media-Key: " "https://" +``` + +You should get a 200 response code once the keys have synced. diff --git a/docs/04.03-upload-image.md b/docs/99-archives/04.03-upload-image.md similarity index 100% rename from docs/04.03-upload-image.md rename to docs/99-archives/04.03-upload-image.md diff --git a/docs/04.04-start-thrall.md b/docs/99-archives/04.04-start-thrall.md similarity index 100% rename from docs/04.04-start-thrall.md rename to docs/99-archives/04.04-start-thrall.md diff --git a/docs/04.05-media-api.md b/docs/99-archives/04.05-media-api.md similarity index 100% rename from docs/04.05-media-api.md rename to docs/99-archives/04.05-media-api.md diff --git a/docs/05-logging.md b/docs/99-archives/05-logging.md similarity index 100% rename from docs/05-logging.md rename to docs/99-archives/05-logging.md diff --git a/docs/99-archives/README.md b/docs/99-archives/README.md new file mode 100644 index 0000000000..60817677f4 --- /dev/null +++ b/docs/99-archives/README.md @@ -0,0 +1,21 @@ +# Grid Docs + +[1 Setting up Cloudformation](./01-cloudformation.md) + +[2.1 Installing DEV requirements](./02.01-dev-setup.md) + +[2.2 Creating configuration](./02.02-configuration.md) + +[3 Running Grid](./03-running.md) + +[4.1 Calling Grid APIs](./04.01-api.md) + +[4.2 Authentication](./04.02-authentication.md) + +[4.3 Uploading an image](./04.03-upload-image.md) + +[4.4 Starting Thrall](./04.04-start-thrall.md) + +[4.5 Media API](./04.05-media-api.md) + +[Migration to Elasticsearch 6](elasticsearch6.md) \ No newline at end of file diff --git a/TROUBLESHOOTING.md b/docs/99-archives/TROUBLESHOOTING.md similarity index 100% rename from TROUBLESHOOTING.md rename to docs/99-archives/TROUBLESHOOTING.md diff --git a/docs/elasticsearch6.md b/docs/99-archives/elasticsearch6.md similarity index 100% rename from docs/elasticsearch6.md rename to docs/99-archives/elasticsearch6.md diff --git a/docs/elasticsearch7.md b/docs/99-archives/elasticsearch7.md similarity index 100% rename from docs/elasticsearch7.md rename to docs/99-archives/elasticsearch7.md diff --git a/docs/local-kinesis.md b/docs/99-archives/local-kinesis.md similarity index 100% rename from docs/local-kinesis.md rename to docs/99-archives/local-kinesis.md diff --git a/docs/README.md b/docs/README.md index 60817677f4..5c32ef3bfa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,21 +1,51 @@ -# Grid Docs +# Table Of Contents +*(Do NOT edit manually. Generated using generate-toc.sh)* + +## [About](00-about/) +- [Vision](00-about/01-vision.md) + +## [Setup](01-setup/) +- [Software dependencies](01-setup/01-software-dependencies.md) +- [AWS dependencies](01-setup/02-aws-dependencies.md) +- [NGINX](01-setup/03-nginx.md) +- [Configuration](01-setup/04-configuration.md) + +## [Running](02-running/) +- [Running locally](02-running/01-running-locally.md) +- [Kahuna](02-running/02-kahuna.md) +- [Accessing logs](02-running/03-logging.md) + +## [Api access](03-api-access/) +- [Authentication](03-api-access/01-authentication.md) + +## [Troubleshooting](04-troubleshooting/) +- [NGINX](04-troubleshooting/01-nginx.md) +- [SBT](04-troubleshooting/02-sbt.md) + +## [Archives](99-archives/) +- [DEV Cloudformation Setup](99-archives/01-cloudformation.md) +- [DEV Setup](99-archives/02.01-dev-setup.md) +- [Configuration](99-archives/02.02-configuration.md) +- [Running services](99-archives/03-running.md) +- [Grid API](99-archives/04.01-api.md) +- [Authentication](99-archives/04.02-authentication.md) +- [Upload Image Directly](99-archives/04.03-upload-image.md) +- [Start Thrall](99-archives/04.04-start-thrall.md) +- [Media API](99-archives/04.05-media-api.md) +- [Logging](99-archives/05-logging.md) +- [Troubleshooting](99-archives/TROUBLESHOOTING.md) +- [Migration to Elasticsearch 6](99-archives/elasticsearch6.md) +- [Migration to Elasticsearch 7](99-archives/elasticsearch7.md) +- [](99-archives/local-kinesis.md) + + +--- +# META: How to create a new documentation file + +## Documentation conventions + +- Find the correct subdirectory your new documentation file belongs to. +- Every documentation file should be markdown (with .md extension). +- First line of every documentation file should contain its title (used to generated the table of content). +- Store all the images in an `/images` subfolder in the same directory the document referencing them will be. -[1 Setting up Cloudformation](./01-cloudformation.md) - -[2.1 Installing DEV requirements](./02.01-dev-setup.md) - -[2.2 Creating configuration](./02.02-configuration.md) - -[3 Running Grid](./03-running.md) - -[4.1 Calling Grid APIs](./04.01-api.md) - -[4.2 Authentication](./04.02-authentication.md) - -[4.3 Uploading an image](./04.03-upload-image.md) - -[4.4 Starting Thrall](./04.04-start-thrall.md) - -[4.5 Media API](./04.05-media-api.md) - -[Migration to Elasticsearch 6](elasticsearch6.md) \ No newline at end of file diff --git a/docs/generate-toc.sh b/docs/generate-toc.sh new file mode 100755 index 0000000000..73170078fe --- /dev/null +++ b/docs/generate-toc.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +cleanDirectoryName() { + # Remove leading number + # Replace dash with space + # Remove trailing '/' + # trim spaces + dirname=$(echo "$1" \ + | sed 's/^[0-9]*//g' \ + | sed 's/-/ /g' \ + | sed 's/\/$//g' \ + | xargs) + + capitalize "$dirname" +} + +cleanFileTitle() { + # Get title + # Remove formatting markdown characters + # Also trim spaces + head -n 1 "$1" | sed 's/[*#]//g' | xargs +} + +capitalize() { + # Capitalize first letter + head=$(echo "$1" | cut -c1 | tr [a-z] [A-Z]) + tail=$(echo "$1" | cut -c2-) + echo "$head$tail" +} + +root=$(git rev-parse --show-toplevel) +docs="$root/docs" +pushd $docs > /dev/null + +generate() { + echo "# Table Of Contents" + echo "*(Do NOT edit manually. Generated using generate-toc.sh)*" + echo "" + for dir in */; do + dirname=$(cleanDirectoryName $dir) + echo "## [$dirname]($dir)" + cd $dir > /dev/null + for doc in *.md; do + if [ "$doc" != "README.md" ]; then + filepath=$dir$doc + title=$(cleanFileTitle $doc) + echo "- [$title]($filepath)" + fi + done + cd .. + echo "" + done + + echo "" + echo "---" + cat how-to-create-a-doc-file.md + echo "" + + popd > /dev/null +} + +generate > README.md diff --git a/docs/how-to-create-a-doc-file.md b/docs/how-to-create-a-doc-file.md new file mode 100644 index 0000000000..3901ad6e7d --- /dev/null +++ b/docs/how-to-create-a-doc-file.md @@ -0,0 +1,8 @@ +# META: How to create a new documentation file + +## Documentation conventions + +- Find the correct subdirectory your new documentation file belongs to. +- Every documentation file should be markdown (with .md extension). +- First line of every documentation file should contain its title (used to generated the table of content). +- Store all the images in an `/images` subfolder in the same directory the document referencing them will be.