Uusi aineistoluku

.gitattributes

@@ -1 +1,15 @@
 *.bat	text eol=crlf working-tree-encoding=cp850
+# Enforce LF line endings for all text files
+* text=auto eol=lf
+
+# Handle line endings for specific file types
+*.bat text eol=crlf
+*.sh text eol=lf
+*.sql text eol=lf
+*.md text eol=lf
+*.json text eol=lf
+*.py text eol=lf
+*.yml text eol=lf
+*.yaml text eol=lf
+*.ini text eol=lf
+*.conf text eol=lf

.gitignore

@@ -39,6 +39,7 @@ htmlcov/
 env
 /*venv/
 .python-version
+.vscode
 
 # Rope project settings
 .ropeproject
@@ -60,8 +61,11 @@ wb-xlate*
 
 # jkr
 /data
+/dump
 muistiot/
 pjh-virheelliset-osoitteet.csv
 *.qgs~
 jkrconfig
 pg_service.conf
+.aider*
+.env

.testenv

@@ -0,0 +1,11 @@
+POSTGRES_PASSWORD=postgres
+
+JKR_DB_HOST=10.43.16.4
+JKR_DB_PORT=5432
+
+JKR_DB=jatehuolto
+JKR_USER=puno_admin
+JKR_PASSWORD=n79FRas7pl01
+PGPASSWORD=n79FRas7pl01
+
+QGIS_BIN_PATH=C:\Program Files\QGIS 3.28.2\bin

.vscode/settings.json

@@ -5,7 +5,7 @@
     "python.formatting.provider": "black",
     "editor.formatOnSave": true,
     "editor.codeActionsOnSave": {
-        "source.organizeImports": true
+        "source.organizeImports": "explicit"
     },
     "python.testing.pytestArgs": [
         "tests"

Dockerfile

@@ -0,0 +1,55 @@
+# Käytä virallista Python 3.11 -slim-kuvaa pohjana
+FROM python:3.11-slim
+
+# Aseta ympäristömuuttujat
+ENV PYTHONUNBUFFERED=1 \
+    POETRY_VERSION=1.5.1 \
+    POETRY_VIRTUALENVS_CREATE=false \
+    APPDATA=/usr/local/bin/dotenv
+
+# Asenna järjestelmäriippuvuudet ja QGIS
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    libssl-dev \
+    libffi-dev \
+    libpq-dev \
+    postgresql-client \
+    gdal-bin \
+    python3-gdal \
+    qgis \
+    libgdal-dev \
+    libspatialindex-dev \
+    curl \
+    git \
+    locales && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Aseta locale
+RUN sed -i '/fi_FI.UTF-8/s/^# //g' /etc/locale.gen && \
+    locale-gen
+ENV LANG fi_FI.UTF-8
+ENV LANGUAGE fi_FI:fi
+ENV LC_ALL fi_FI.UTF-8
+
+# Set GDAL configuration paths
+ENV CPLUS_INCLUDE_PATH=/usr/include/gdal
+ENV C_INCLUDE_PATH=/usr/include/gdal
+
+# Asenna Poetry
+RUN curl -sSL https://install.python-poetry.org | python3 -
+
+# Lisää Poetry PATH:iin
+ENV PATH="/root/.local/bin:$PATH"
+
+# Päivitä pip
+RUN pip install --upgrade pip
+
+# Avaa portit tarvittaessa (esim. 8000 jos sovellukselle)
+EXPOSE 8000
+
+# Määritä työhakemisto
+WORKDIR /app
+
+# Aseta oletuskomento: asenna riippuvuudet, siirry scripts-kansioon ja käynnistä bash
+CMD ["bash", "-c", "poetry install --no-root && cd scripts && exec /bin/bash"]

README.md

@@ -4,21 +4,23 @@ This repository contains core parts of Jätteenkuljetusrekisteri including the d
 
 This project is intended to be extended by customer specific modifications etc. import format plugins etc.
 
-
 ## Installing
 
 You probably want install this to a virtual environment so your system python don't get cluttered of dependencies.
+
 ```bash
 $ python -m venv jkr-venv
 $ jkr-venv\scripts\activate
 ```
 
 Download the whl file etc. `jkr-0.1.0-py3-none-any.whl`
+
 ```bash
 (jkr-venv) $ pip install jkr-0.1.0-py3-none-any.whl
 ```
 
 Now import script can be started
+
 ```bash
 (jkr-venv) $ jkr import SIIRTOTIEDOSTO TIEDONTUOTTAJA
 ```
@@ -28,7 +30,7 @@ Now import script can be started
 The development environment uses [Poetry](https://python-poetry.org/). Install it before anything.
 
 ```bash
-$ git clone https://github.com/GispoCoding/jkr-core.git
+$ git clone https://github.com/punosmobile/jkr-core.git
 $ cd jkr-core
 
 $ poetry install
@@ -39,32 +41,55 @@ $ poetry install
 Install docker and docker-compose (version >= 1.28.0)
 
 Copy .env.template to %APPDATA%/jkr/.env and change parameters
+
 ```bash
 $ cp .env.template .env
 
 # Edit the .env file
 $ nano .env
 ```
 
+Start database
+
 ```bash
-docker-compose --env-file "${env:APPDATA}/jkr/.env" up db -d
-docker-compose --env-file "${env:APPDATA}/jkr/.env" up flyway
+docker-compose -f dev.docker-compose.yml --env-file ".env.local" up db -d
+```
+
+Run migrations
+
+```bash
+docker-compose -f dev.docker-compose.yml --env-file ".env.local" up flyway
+```
+
+Optionally Scripts can be runned in container. First build image
+
+```bash
+docker build -t jkr-core-runner:latest -f Dockerfile
+```
+
+Start script-runner container
+
+```bash
+ docker-compose -f dev.docker-compose.yml --env-file ".env.local" run jkr-core-runner
 ```
 
 ## Handling database model changes
+
 The diff operation in pgModeler is quite fragile and not recommended used directly.
 
 We use Flyway (for now) to generate database migrations. Flyway is a SQL file based migration system.
 PgModeler is used to generate diff SQL files and then Flyway is used to apply the migrations.
 
 ### Create the initial migration
+
 ```bash
 pgmodeler-cli.exe --export-to-file --input jkr.dbm --output migrations/V1__initial.sql --pgsql-ver 12.0
 ```
 
 ### Creating incremental migrations
+
 ```bash
-# 1. Verify db is up to date (all current migrations applied) by running 
+# 1. Verify db is up to date (all current migrations applied) by running
 docker-compose --env-file "${env:APPDATA}/jkr/.env" up flyway
 
 # 2. Modify the pgmodeler model
@@ -74,18 +99,20 @@ pgmodeler-cli.exe --diff --save --input jkr.dbm --compare-to jkr --conn-alias lo
 
 # 4. Validate and modify the migration file
 
-# 5. Migrate by running 
+# 5. Migrate by running
 docker-compose --env-file "${env:APPDATA}/jkr/.env" up flyway
 
 # 6. Verify all changes in model are apllied to the database by executing step 3 again and checking that no diff is generated.
 ```
 
 **Repairing migrations**
+
 ```bash
 docker-compose --env-file "${env:APPDATA}/jkr/.env" run --rm flyway repair
 ```
 
 ### Running migrations against different platform
+
 ```bash
 docker-compose run --rm flyway migrate -user=jkr_admin -password=<password> -url=jdbc:postgresql://trepx-paikka1.tre.t.verkko:5432/ymparisto_test_db
 ```
@@ -95,33 +122,34 @@ docker-compose run --rm flyway migrate -user=jkr_admin -password=<password> -url
 The QGIS project reads data from a PostgreSQL service named `jkr` with a QGIS authentication which id is `jkruser`.
 
 1. Create a PostgreSQL service file for each environment (Development, Testing, Production) to some folder for example in `<your home folder>/jkrconfig/`. Name the files for example `pg_service_jkr_dev.conf`, `pg_service_jkr_test.conf`, `pg_service_jkr_prod.conf`. Add the following with correct values for each environment:
+
 ```ini
 [jkr]
 host=localhost
 port=5435
 dbname=ymparisto_db
 ```
-2. Create a QGIS-profile for each environment (Development, Testing, Production). Name the profiles for example `jkr-dev`, `jkr-test`, `jkr-prod`. A new QGIS window will open. Use that   
-![schreenshot of new profile menu](docs/img/qgis-new-profile.png)
+
+2. Create a QGIS-profile for each environment (Development, Testing, Production). Name the profiles for example `jkr-dev`, `jkr-test`, `jkr-prod`. A new QGIS window will open. Use that  
+   ![schreenshot of new profile menu](docs/img/qgis-new-profile.png)
 3. In QGIS settings add a `PGSERVICEFILE` environment variable and fill the file path of corresponding service file as a value.  
-![screenshot of menu location](docs/img/qgis-settings.png)  
-![screenshot of the setting dialog](docs/img/qgis-pgservicefile-environment-variable.png)
+   ![screenshot of menu location](docs/img/qgis-settings.png)  
+   ![screenshot of the setting dialog](docs/img/qgis-pgservicefile-environment-variable.png)
 4. Restart QGIS to make the environment variable to take effect.
 5. Create a authentication key to QGIS which ID is `jkruser`.  
-![screenshot of the authentication dialog](docs/img/qgis-authentication.png)
+   ![screenshot of the authentication dialog](docs/img/qgis-authentication.png)
 6. Create a new PostgreSQL connection  
-![screenshot of the new connection menu](docs/img/qgis-new-connection.png)  
-![screenshot of the new connection dialog](docs/img/qgis-create-connection.png)
+   ![screenshot of the new connection menu](docs/img/qgis-new-connection.png)  
+   ![screenshot of the new connection dialog](docs/img/qgis-create-connection.png)
 7. Open the QGIS project from the jkr-qgis-projektit -schema.  
-![screenshot of the qgis projects schema](docs/img/qgis-open-project.png)
-
+   ![screenshot of the qgis projects schema](docs/img/qgis-open-project.png)
 
 > **Development**  
 > For development use the [QGIS-project](qgis/jkr.qgs) can be used.
 
 ## Using jkr single command importer
 
-In jkr-core/scripts/jkr_posti.sql, replace  ```<POSTI>``` with the path to your posti file.
+In jkr-core/scripts/jkr_posti.sql, replace `<POSTI>` with the path to your posti file.
 
 In jkr-core/scripts/import_and_create_kohteet.bat, replace the following lines to match your database connection:
 
@@ -137,12 +165,13 @@ The single command importer can be called with the following from the root.
 ```
 jkr import_and_create_kohteet <POIMINTAPVM> <DVV> <PERUSMAKSU> <POSTI>
 ```
+
 1. Replace <POIMINTAPVM> with the poimintapäivämäärä of DVV file you're about to import. Required.
 1. Replace <DVV> with the filepath of DVV file you're about to import. Required.
 1. Replace <PERUSMAKSU> with the filepath of perusmaksurekiseteri file you want to use.
-If you do not want to use perusmaksurekisteri, leave <PERUSMAKSU> out of the command.
+   If you do not want to use perusmaksurekisteri, leave <PERUSMAKSU> out of the command.
 1. Replace <POSTI> with "posti" (without quotation marks) if you want to import posti data.
-If you do not want to import posti data, leave <POSTI> out of the command.
+   If you do not want to import posti data, leave <POSTI> out of the command.
 
 ## Testing
 
@@ -162,8 +191,8 @@ The data used in tests (`/tests/data`) is mostly dummy data created only for tes
 
 #### Postal code data
 
-The postal code data (`/tests/data/test_data_import`) is real data downloaded from Postal Code Services by Posti. Please see the current service description and terms of use if you share this data further. [Service description and terms of use](https://www.posti.fi/mzj3zpe8qb7p/1eKbwM2WAEY5AuGi5TrSZ7/c76a865cf5feb2c527a114b8615e9580/posti-postal-code-services-service-description-and-terms-of-use-20150101.pdf) 
+The postal code data (`/tests/data/test_data_import`) is real data downloaded from Postal Code Services by Posti. Please see the current service description and terms of use if you share this data further. [Service description and terms of use](https://www.posti.fi/mzj3zpe8qb7p/1eKbwM2WAEY5AuGi5TrSZ7/c76a865cf5feb2c527a114b8615e9580/posti-postal-code-services-service-description-and-terms-of-use-20150101.pdf)
 
 ## Naming development branches
 
-Because this repository is developed mostly in customer specific projects the label of the project may be good to be included in the branch name. The preferred naming convention is `{label-of-project}-{issue-in-that-project}-{description}`. For example, `"Lahti-99-kuljetustietojen-tallennus"`. Please avoid umlauts and use hyphens as separators. 
+Because this repository is developed mostly in customer specific projects the label of the project may be good to be included in the branch name. The preferred naming convention is `{label-of-project}-{issue-in-that-project}-{description}`. For example, `"Lahti-99-kuljetustietojen-tallennus"`. Please avoid umlauts and use hyphens as separators.

db/migrations/V2.43_0__Add_rakennusluokka_2018.sql

@@ -0,0 +1,9 @@
+-- V2.43.0__Add_rakennusluokka_2018.sql
+ALTER TABLE jkr.rakennus 
+ADD COLUMN rakennusluokka_2018 VARCHAR(4);
+
+COMMENT ON COLUMN jkr.rakennus.rakennusluokka_2018 
+IS 'Rakennusluokka 2018 -luokituksen mukainen rakennuksen käyttötarkoitus';
+
+CREATE INDEX idx_rakennus_rakennusluokka_2018 
+ON jkr.rakennus(rakennusluokka_2018);

db/migrations/V2.44_0__Add_indexes_to_speed_up_geospatial_search.sql

@@ -0,0 +1,46 @@
+-- V2.44.0__Add_spatial_indexes.sql
+-- Luodaan spatiaaliset indeksit rakennusten geometrioille
+CREATE INDEX IF NOT EXISTS idx_rakennus_geom 
+ON jkr.rakennus USING GIST (geom);
+
+-- Luodaan indeksit yleisimmin käytetyille hakukentille
+CREATE INDEX IF NOT EXISTS idx_rakennus_omistaja 
+ON jkr.rakennuksen_omistajat (osapuoli_id);
+
+CREATE INDEX IF NOT EXISTS idx_rakennus_osoite 
+ON jkr.osoite (rakennus_id, katu_id, osoitenumero);
+
+-- Käytetään oikeaa sarakenimeä rakennuksenkayttotarkoitus_koodi
+CREATE INDEX IF NOT EXISTS idx_rakennuksenkayttotarkoitus 
+ON jkr.rakennus (rakennuksenkayttotarkoitus_koodi);
+
+-- Luodaan materialisoitu näkymä lähellä olevien rakennusten löytämiseen
+CREATE MATERIALIZED VIEW IF NOT EXISTS jkr.nearby_buildings AS
+SELECT 
+    r1.id as rakennus1_id,
+    r2.id as rakennus2_id,
+    ST_Distance(r1.geom, r2.geom) as distance
+FROM jkr.rakennus r1
+JOIN jkr.rakennus r2 ON 
+    r1.id < r2.id AND
+    ST_DWithin(r1.geom, r2.geom, 300)
+WHERE
+    r1.geom IS NOT NULL AND 
+    r2.geom IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_nearby_buildings_r1 
+ON jkr.nearby_buildings (rakennus1_id);
+
+CREATE INDEX IF NOT EXISTS idx_nearby_buildings_r2 
+ON jkr.nearby_buildings (rakennus2_id);
+
+CREATE INDEX IF NOT EXISTS idx_nearby_buildings_dist 
+ON jkr.nearby_buildings (distance);
+
+-- Funktio materialisoidun näkymän päivittämiseen
+CREATE OR REPLACE FUNCTION jkr.refresh_nearby_buildings()
+RETURNS void AS $$
+BEGIN
+    REFRESH MATERIALIZED VIEW jkr.nearby_buildings;
+END;
+$$ LANGUAGE plpgsql;

db/migrations/V2.45_0__Add_indexes_to_speed_up_kohde_update.sql

@@ -0,0 +1,4 @@
+-- V2.45.0__Add_spatial_indexes.sql
+-- Luodaan indeksit nopeampaa päivitystä varten
+CREATE INDEX idx_kohteen_rakennukset_rakennus_id ON jkr.kohteen_rakennukset(rakennus_id);
+CREATE INDEX idx_kohteen_osapuolet_osapuoli_rooli ON jkr.kohteen_osapuolet(osapuoli_id, osapuolenrooli_id);