powerbi: Add setup task. Download Dockerfiles as part of build target…

…s. Update docs.

docs:
- Add comments to config.mk.
- Mention assumption about current directory.
- database: Mention DATABASE_* configurations. Describe passwordless login.
- filesystem: Mention CARDINAL_WORKDIR configuration. Describe file ownership.
- build: Document current directory requirements. List target's effects.
- cron: Mention CARDINAL_WORKDIR configuration. Describe crontab ownership.
- clean: Add section.

powerbi/Makefile

@@ -11,12 +11,27 @@ FORCE:
 
 include config.mk
 
+config.mk:
+	curl -sSLO https://raw.githubusercontent.com/open-contracting/bi.open-contracting.org/refs/heads/main/powerbi/config.mk
+
+cron.sh:
+	curl -sSLO https://github.com/open-contracting/bi.open-contracting.org/raw/refs/heads/main/powerbi/cron.sh
+
+.PHONY: setup
+setup: config.mk cron.sh
+
 kingfisher-collect:
 	git clone https://github.com/open-contracting/kingfisher-collect.git
 
 cardinal-rs:
 	git clone https://github.com/open-contracting/cardinal-rs.git
 
+Dockerfile_cardinal:
+	curl -sSLO https://raw.githubusercontent.com/open-contracting/bi.open-contracting.org/refs/heads/main/powerbi/Dockerfile_cardinal
+
+Dockerfile_python:
+	curl -sSLO https://raw.githubusercontent.com/open-contracting/bi.open-contracting.org/refs/heads/main/powerbi/Dockerfile_python
+
 .PHONY: pull-kingfisher-collect
 pull-kingfisher-collect: kingfisher-collect
 	git -C kingfisher-collect pull --rebase
@@ -26,11 +41,11 @@ pull-cardinal-rs: cardinal-rs
 	git -C cardinal-rs pull --rebase
 
 .PHONY: build-python
-build-python: pull-kingfisher-collect pull-cardinal-rs
+build-python: pull-kingfisher-collect pull-cardinal-rs Dockerfile_python
 	docker build --file Dockerfile_python --tag kingfisher-collect .
 
 .PHONY: build-cardinal
-build-cardinal: pull-cardinal-rs
+build-cardinal: pull-cardinal-rs Dockerfile_cardinal
 	docker build --file Dockerfile_cardinal --tag cardinal-rs .
 
 .PHONY: build
@@ -123,14 +138,14 @@ filesystem: data logs scratch ecuador_sercop_bulk.ini
 print-crontab:
 	printf "15 0 * * * CARDINAL_DBNAME=$(DATABASE_NAME) CARDINAL_DBUSER=$(DATABASE_USER) CARDINAL_DBHOST=$(DATABASE_HOST) CARDINAL_DBHOST_DOCKER=$(DATABASE_HOST_DOCKER) $(CARDINAL_WORKDIR)/cron.sh"
 
-.PHONY: clean
-clean:
+.PHONY: clean-build
+clean-build:
 	rm -rf kingfisher-collect
 	rm -rf cardinal-rs
-	rm -f ecuador_sercop_bulk.ini
 
 .PHONY: force-clean
-force-clean: clean
+force-clean: clean-build
+	rm -f ecuador_sercop_bulk.ini
 	rm -rf data
 	rm -rf logs
 	rm -rf scratch

powerbi/README.md

@@ -1,23 +1,43 @@
 # Power BI
 
-This directory contains files to support deploying [Kingfisher Collect](https://kingfisher-collect.readthedocs.io/en/latest/) and [Cardinal](https://cardinal.readthedocs.io/en/latest/) using Docker.
+Follow these instructions to deploy [Kingfisher Collect](https://kingfisher-collect.readthedocs.io/en/latest/) and [Cardinal](https://cardinal.readthedocs.io/en/latest/) using Docker.
 
 The `Makefile` makes this easy to setup. You can configure it by changing the variables in `config.mk`.
 
+All commands assume that the current directory is the "working directory" for the project. The local user must have read, write and execute permissions to the working directory.
+
+## Tips
+
 To print the commands that a `make` target would execute, use the `-n` (`--dry-run`) option. For example:
 
 ```bash
 make -n database
 ```
 
-To run all targets (setup the database and filesystem, build the images, and install the crontab), run:
+To run all targets (setup the database, setup the filesystem, build the images, and install the crontab), run:
 
 ```bash
 make -s
 make -s print-crontab | crontab
 ```
 
-## PostgreSQL
+## Setup
+
+Download the `Makefile` to the current directory:
+
+```bash
+curl -sSLO https://raw.githubusercontent.com/open-contracting/bi.open-contracting.org/refs/heads/main/powerbi/Makefile
+```
+
+Download the `config.mk` and `cron.sh` files to the current directory, if they don't exist:
+
+```bash
+make setup
+```
+
+Lastly, edit the `config.mk` file, as needed.
+
+## Database
 
 Run `make database` as a local user with the [CREATEDB](https://www.postgresql.org/docs/current/sql-createrole.html) privilege (for example, as the `postgres` user):
 
@@ -27,42 +47,63 @@ make -s database
 
 This will:
 
-- Create a database (`cardinal`, by default), if it doesn't exist
-- Create a user (`cardinal`, by default), if it doesn't exist
-- Create the `ecuador_sercop_bulk_result` table, owned by the user, if it doesn't exist
-- Create (or re-create) the `codelist`, `indicator` and `cpc` tables, owned by the user
+- Create a PostgreSQL database (the `DATABASE_NAME` configuration, by default `cardinal`), if it doesn't exist
+- Create a PostgreSQL user (the `DATABASE_USER` configuration, by default `cardinal`), if it doesn't exist
+- Create the `ecuador_sercop_bulk_result` table, owned by the PostgreSQL user, if it doesn't exist
+- Create (or re-create) the `codelist`, `indicator` and `cpc` tables, owned by the PostgreSQL user
+
+The PostgreSQL host is set by the `DATABASE_HOST` configuration, by default `localhost`.
+
+This assumes that the local user can authenticate with PostgreSQL without a password. This can be done in a few ways:
+
+1. Configure the [pg_bha.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) file to allow the local user to login as the PostgreSQL user with the same name. For example, for the `postgres` user:
+
+  ```none
+  local all postgres peer
+  ```
+
+1. Create a [.pgpass](https://www.postgresql.org/docs/current/libpq-pgpass.html) file in the local user's home directory. For example:
+
+  ```none
+  localhost:5432:cardinal:cardinal:strong-password
+  ```
 
 ## Filesystem
 
-Run `make filesystem` from the working directory for the project.
+Run `make filesystem` from the working directory (the `CARDINAL_WORKDIR` configuration) for the project, as the local user that will run the cron job:
 
 ```bash
 make -s filesystem
 ```
 
 This will:
 
-- Create `data`, `logs` and `scratch` directories
-- Download Cardinal's settings file to `ecuador_sercop_bulk.ini`
+- Create `data`, `logs` and `scratch` directories, owned by the local user, if they don't exist
+- Download Cardinal's settings file to `ecuador_sercop_bulk.ini`, owned by the local user
 
 ## Docker
 
-Run `make build` to build two images:
+Run `make build` from any directory (that contains the `Makefile`).
 
-- `kingfisher-collect`, for running `scrapy` and `manage.py` commands, like:
+This will:
 
-  ```bash
-  docker run --rm --name kingfisher-collect kingfisher-collect scrapy --help
-  docker run --rm --name kingfisher-collect kingfisher-collect python manage.py --help
-  ```
+- Clone the `kingfisher-collect` and `cardinal-rs` repositories into the current directory, if they don't exist
+- Download the `Dockerfile_cardinal` and `Dockerfile_python` files to the current directory, if they don't exist
+- Pull changes to the `kingfisher-collect` and `cardinal-rs` repositories
+- Build the `kingfisher-collect` and `cardinal-rs` images
 
-- `cardinal`, for running `ocdscardinal` commands, like
+The `kingfisher-collect` image is for running `scrapy` and `manage.py` commands, like:
 
-  ```bash
-  docker run --rm --name cardinal-rs cardinal-rs --help
-  ```
+```bash
+docker run --rm --name kingfisher-collect kingfisher-collect scrapy --help
+docker run --rm --name kingfisher-collect kingfisher-collect python manage.py --help
+```
 
-This clones the `kingfisher-collect` and `cardinal-rs` repositories into the current directory.
+The `cardinal-rs` image is for running `ocdscardinal` commands, like:
+
+```bash
+docker run --rm --name cardinal-rs cardinal-rs --help
+```
 
 ## Cron
 
@@ -72,12 +113,28 @@ Preview the crontab entry:
 make -s print-crontab
 ```
 
-Add the crontab entry:
+Make sure the directory of the `cron.sh` file in the crontab entry is correct (if not, edit the `CARDINAL_WORKDIR` configuration).
+
+Add the crontab entry to the local user's crontab file:
 
 ```bash
 make -s print-crontab | crontab
 ```
 
+## Clean
+
+If desired, you can delete the `kingfisher-collect` and `cardinal-rs` directories, which are downloaded by the `build` target, but aren't needed after building images:
+
+```bash
+make clean-build
+```
+
+If you need to start over, delete the cron job manually. Then, delete the `ecuador_sercop_bulk.ini` file and the `data`, `logs` and `scratch` directories:
+
+```bash
+make force-clean
+```
+
 ## Reference
 
 This process replicates the configuration in the [incremental](https://github.com/open-contracting/deploy/blob/main/salt/kingfisher/collect/incremental.sls) state from the [deploy](https://ocdsdeploy.readthedocs.io/en/latest/) repository.

powerbi/config.mk

@@ -1,5 +1,10 @@
+# The host of the database used for this project.
+DATABASE_HOST=localhost
+# The name of the database used for this project.
 DATABASE_NAME=cardinal
+# The database user for this project.
 DATABASE_USER=cardinal
-DATABASE_HOST=localhost
+# The host of the database, from within the Docker container.
 DATABASE_HOST_DOCKER=host.docker.internal
+# The working directory for this project.
 CARDINAL_WORKDIR=/absolute/path/workdir