reorganize demo && tests

.github/workflows/playwright.yml

@@ -0,0 +1,27 @@
+name: Playwright Tests
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: lts/*
+    - name: Install dependencies
+      run: npm ci
+    - name: Install Playwright Browsers
+      run: npx playwright install --with-deps
+    - name: Run Playwright tests
+      run: npx playwright test
+    - uses: actions/upload-artifact@v4
+      if: ${{ !cancelled() }}
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 30

.gitignore

@@ -1,2 +1,6 @@
 node_modules
-.parcel-cache
+.parcel-cache
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

src/bfsm.ts → demo/bfsm.ts

@@ -69,8 +69,8 @@ export function packBfsm(items: any[]): { data: BinaryFeatureStateMap, transfer:
   };
 }
 
-export function unpackBfsm(binaryMap: BinaryFeatureStateMap): Map<string, Record<string, any>> {
-    const result = new Map<string, Record<string, any>>();
+export function unpackBfsm(binaryMap: BinaryFeatureStateMap): any[] {
+    const result: any[] = [];
 
     const { attributes, size } = binaryMap;
 
@@ -98,17 +98,15 @@ export function unpackBfsm(binaryMap: BinaryFeatureStateMap): Map<string, Record
     }
 
     for (let i = 0; i < size; i++) {
-        // TODO https://jira.2gis.ru/browse/TILES-6351 Поддержать строковые идентификаторы, сейчас работает только с внутренними id карты
         const id = 
-            String(idAttribute.view[(i * idAttribute.stride) / 4]) +
-            String(idAttribute.view[(i * idAttribute.stride) / 4 + 1]);
+            String(idAttribute.view[(i * idAttribute.stride) / 4]);
 
         const featureState = {};
         for (const attribute of otherAttributes) {
             const value = attribute.view[(i * attribute.stride) / 4];
             featureState[attribute.name] = !Number.isNaN(value) ? value : null;
         }
-        result.set(id, featureState);
+        result.push({ id, ...featureState });
     }
 
     return result;

demo/config.ts

@@ -0,0 +1,94 @@
+import { packJson, unpackJson, PackMethod, JsonType } from "../src/json2pbf";
+import { packBfsm, unpackBfsm } from "./bfsm";
+
+function makeArray (length: number) {
+    const arr: any[] = [];
+    for (let i = 0; i < length; i++) {
+        arr.push({ id: Math.trunc(Math.random() * 10 ** 9).toString(), hidden: Math.random() > 0.5 })
+    }
+    return arr;
+}
+
+function makeBfsmArray (length: number) {
+    const arr: any[] = [];
+    for (let i = 0; i < length; i++) {
+        arr.push({ id: Math.trunc(Math.random() * 10 ** 9).toString(), hidden: Math.round(Math.random()) })
+    }
+    return arr;
+}
+
+function makeColArray (length: number) {
+    const arr: any = { id: [], hidden: [] };
+    for (let i = 0; i < length; i++) {
+        arr.id.push(Math.trunc(Math.random() * 10 ** 9).toString());
+        arr.hidden.push(Math.random() > 0.5)
+    }
+    return arr;
+}
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder('utf-8');
+
+type Codec = { name: string, makeArray: (length: number) => any, pack: (data: any) => any, unpack: (data: any) => any };
+
+export const codecs: Codec[] = [
+    {
+        name: 'as is (no serialization)',
+        pack: (data) => ({ data }),
+        unpack: (data) => ({ data }),
+        makeArray,
+    },
+    {
+        name: 'string (JSON.stringify / JSON.parse)',
+        pack: (data) => ({ data: JSON.stringify(data) }),
+        unpack: (data) => ({ data: JSON.parse(data) }),
+        makeArray,
+    },
+    {
+        name: 'JSON.stringify + TextEncoder / JSON.parse + TextDecoder',
+        pack: (data) => ({ data: encoder.encode(JSON.stringify(data)) }),
+        unpack: (data) => {
+            const hash = decoder.decode(data);
+            return { data: JSON.parse(hash) };
+        },
+        makeArray
+    },
+    {
+        name: 'json2pbf PackMethod.Generic',
+        pack: (data) => ({ data: packJson(data) }),
+        unpack: (data) => {
+            const unpacked = unpackJson(data);
+            return { data: unpacked };
+        },
+        makeArray
+    },
+    {
+        name: 'json2pbf PackMethod.Row',
+        pack: (data) => ({
+            data: packJson(data, { method: PackMethod.Row, columns: { id: JsonType.String, hidden: JsonType.Boolean } })
+        }),
+        unpack: (data) => {
+            const unpacked = unpackJson(data);
+            return { data: unpacked };
+        },
+        makeArray
+    },
+    {
+        name: 'json2pbf PackMethod.Columnar',
+        pack: (data) => ({ data: packJson(data, { method: PackMethod.Columnar, columns: { id: JsonType.String, hidden: JsonType.Boolean } }) }), 
+        unpack: (data) => {
+            const unpacked = unpackJson(data);
+            return { data: unpacked };
+        },
+        makeArray: makeColArray
+    },
+    {
+        name: 'BFSM',
+        pack: data => packBfsm(data),
+        unpack: (data) => {
+            const unpacked = unpackBfsm(data);
+            return { data: unpacked };
+        },
+        makeArray: makeBfsmArray
+    },
+]

demo/index.ts

@@ -0,0 +1,96 @@
+import { codecs } from './config';
+
+const url = new URL(location.href);
+const isDemo = url.searchParams.get('test') === null;
+
+const worker = new Worker(new URL('./worker.ts', import.meta.url), {
+    type: 'module',
+  });
+
+const t = () => performance.now();
+
+async function performTest(type: number, arraySize: number, iterations: number) {
+
+    const codec = codecs[type];
+
+    function waitResults() {
+        const results: any[] = [];
+        const converted: any[] = [];
+        return new Promise<any>((res) => {
+            let count = 0;
+            let receiveStart = NaN;
+            let receiveTime = NaN;
+            let unpackStart = NaN;
+            let unpackTime = NaN;
+            let statsFromWorker: any = undefined;
+
+            worker.onmessage = (evt) => {
+                if (count === 0) {
+                    receiveStart = t();
+                }
+                if (!evt.data.data) {
+                    statsFromWorker = evt.data;
+                    return;
+                }
+                count++;
+                results.push(evt.data.data);
+                if (count === iterations) {
+                    receiveTime = t() - receiveStart;
+
+                    unpackStart = t();
+                    for (let i = 0; i < iterations; i++) {
+                        converted.push(codec.unpack(results[i]));
+                    }
+                    unpackTime= t() - unpackStart;
+                    // выводим последний элемент что бы проверить на валидность
+                    const last = converted[converted.length - 1].data;
+                    setTimeout(() => res({ last, ...statsFromWorker, receiveTime, unpackTime }) , 300);
+                }
+            };
+            worker.postMessage({ codec: type, iterations });
+        });
+    }
+
+    const testData = codec.makeArray(arraySize);
+    worker.postMessage({ testData });
+    const { last, ...stats } = await waitResults();
+    const equal = JSON.stringify(last) === JSON.stringify(testData);
+
+    if (!equal) {
+        console.log(last, testData)
+    }
+    return { equal, ...stats };
+}
+
+(window as any).performTest = performTest;
+
+async function runDemo () {
+    const configurations = [
+        { arraySize: 10**4, iterations: 100 },
+        { arraySize: 10**5, iterations: 100 },
+        { arraySize: 10**6, iterations: 10 },
+        { arraySize: 10**7, iterations: 1 },
+    ];
+    for (const { arraySize, iterations } of configurations) {
+        const h4 = document.createElement('h4');
+        h4.innerText = `Array size: ${arraySize}, iterations: ${iterations}`;
+        document.body.appendChild(h4);
+        for (let i = 0; i < codecs.length; i++) {
+            const div = document.createElement('div');
+            div.innerText = codecs[i].name;
+            document.body.appendChild(div);
+            const pre = document.createElement('pre');
+            try {
+                pre.innerText = JSON.stringify(await performTest(i, arraySize, iterations), null, 2);
+            } catch (err) {
+                pre.innerText = '[ERROR] ' + err.message;
+            }
+            document.body.appendChild(pre);
+        }
+    }          
+}
+
+
+if (isDemo) {
+    runDemo();
+}

demo/utils.ts

@@ -0,0 +1,25 @@
+import { unpackJson } from "../src/json2pbf";
+
+export function unpackAndBuildMap(data: ArrayBuffer, columnar = false) {
+    const unpacked = unpackJson(data);
+    const map = new Map();
+    if (columnar) {
+        console.log(unpacked)
+        for (let i = 0; i < unpacked.id.length; i++) {
+            const featureState = {
+                hidden: unpacked.hidden[i]
+            }
+            map.set(unpacked.id[i], featureState);
+
+        }
+    } else {
+        for (let i = 0; i < unpacked.length; i++) {
+            const item = unpacked[i];
+            const featureState = {
+                hidden: item.hidden
+            }
+            map.set(item.id, featureState);
+        }    
+    }
+    return map;
+}

demo/worker.ts

@@ -0,0 +1,55 @@
+import { codecs } from './config';
+
+const t = () => performance.now();
+
+let testData = null;
+
+onmessage = function ({ data }) {
+    if (data.testData) {
+        testData = data.testData;
+        return;
+    }
+
+    if (data.codec !== undefined) {
+        const codec = codecs[data.codec];
+        const iterations = data.iterations;
+        const results = new Array<any>(iterations);
+        const packStart = t();
+        try {
+            for (let i = 0; i < iterations; i++) {
+                results[i] = codec.pack(testData);
+            }    
+        } catch (err) {
+            for (let i = 0; i < iterations; i++) {
+                results[i] = { data: [] }
+            }
+            console.error(err);
+        }
+        const packTime = t() - packStart;
+
+        let binarySize = NaN;
+
+        if (results[0].data instanceof ArrayBuffer) {
+            binarySize = results[0].data.byteLength;
+        } else if ('transfer' in results[0]) {
+            binarySize = results[0].transfer[0].byteLength;
+        }
+
+        const sendStart = t();
+        for (let i = 0; i < iterations; i++) {
+            if (data instanceof ArrayBuffer) {
+                this.postMessage(results[i], { transfer: [data] });
+            } else if (data instanceof Float64Array) {
+                this.postMessage(results[i], { transfer: [data.buffer] });
+            } else if ('transfers' in results[i]) {
+                const { data, transfer } = results[i];
+                this.postMessage({ data }, { transfer });
+            } else {
+                this.postMessage(results[i]);
+            }
+        }
+        const sendTime= t() - sendStart;
+
+        this.postMessage({ packTime, sendTime, binarySize });
+    }
+};