From efbfce93623c2012c7e3d18cbe326b9d60aa1027 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= <joern.berkefeld@accenture.com>
Date: Mon, 12 Apr 2021 16:46:51 +0200
Subject: [PATCH 1/8] #9 first draft at adding filter definitions

---
 lib/MetadataTypeDefinitions.js                |   1 +
 lib/MetadataTypeInfo.js                       |   1 +
 lib/metadataTypes/Filter.js                   | 193 +++++++++++-
 lib/metadataTypes/FilterDefinition.js         | 279 ++++++++++++++++++
 .../definitions/Filter.definition.js          |  70 ++---
 .../FilterDefinition.definition.js            | 113 +++++++
 6 files changed, 622 insertions(+), 35 deletions(-)
 create mode 100644 lib/metadataTypes/FilterDefinition.js
 create mode 100644 lib/metadataTypes/definitions/FilterDefinition.definition.js

diff --git a/lib/MetadataTypeDefinitions.js b/lib/MetadataTypeDefinitions.js
index d6b931e75..01a08aeb4 100644
--- a/lib/MetadataTypeDefinitions.js
+++ b/lib/MetadataTypeDefinitions.js
@@ -20,6 +20,7 @@ const MetadataTypeDefinitions = {
     eventDefinition: require('./metadataTypes/definitions/EventDefinition.definition'),
     fileTransfer: require('./metadataTypes/definitions/FileTransfer.definition'),
     filter: require('./metadataTypes/definitions/Filter.definition'),
+    filterDefinition: require('./metadataTypes/definitions/FilterDefinition.definition'),
     folder: require('./metadataTypes/definitions/Folder.definition'),
     ftpLocation: require('./metadataTypes/definitions/FtpLocation.definition'),
     importFile: require('./metadataTypes/definitions/ImportFile.definition'),
diff --git a/lib/MetadataTypeInfo.js b/lib/MetadataTypeInfo.js
index 594af4d4e..8e0d19ed2 100644
--- a/lib/MetadataTypeInfo.js
+++ b/lib/MetadataTypeInfo.js
@@ -20,6 +20,7 @@ const MetadataTypeInfo = {
     eventDefinition: require('./metadataTypes/EventDefinition'),
     fileTransfer: require('./metadataTypes/FileTransfer'),
     filter: require('./metadataTypes/Filter'),
+    filterDefinition: require('./metadataTypes/FilterDefinition'),
     folder: require('./metadataTypes/Folder'),
     ftpLocation: require('./metadataTypes/FtpLocation'),
     importFile: require('./metadataTypes/ImportFile'),
diff --git a/lib/metadataTypes/Filter.js b/lib/metadataTypes/Filter.js
index 658fa493d..b45000ff4 100644
--- a/lib/metadataTypes/Filter.js
+++ b/lib/metadataTypes/Filter.js
@@ -1,6 +1,32 @@
 'use strict';
 
+/**
+ * @typedef {Object} FilterItem
+ * @property {number} categoryId folder id
+ * @property {string} [createdDate] -
+ * @property {string} customerKey key
+ * @property {string} destinationObjectId DE/List ID
+ * @property {1|2|3|4} destinationTypeId 1:SubscriberList, 2:DataExtension, 3:GroupWizard, 4:BehavioralData
+ * @property {string} filterActivityId ?
+ * @property {string} filterDefinitionId ObjectID of filterDefinition
+ * @property {string} modifiedDate -
+ * @property {string} name name
+ * @property {string} sourceObjectId DE/List ID
+ * @property {1|2|3|4} sourceTypeId 1:SubscriberList, 2:DataExtension, 3:GroupWizard, 4:BehavioralData
+ * @property {number} statusId ?
+ *
+ * @typedef {Object.<string, FilterItem>} FilterMap
+ */
+
 const MetadataType = require('./MetadataType');
+const Util = require('../util/util');
+
+const dataTypes = {
+    1: 'List',
+    2: 'DataExtension',
+    3: 'Group Wizard',
+    4: 'Behavioral Data',
+};
 
 /**
  * Filter MetadataType
@@ -13,11 +39,176 @@ class Filter extends MetadataType {
      * but only with some of the fields. So it is needed to loop over
      * Filters with the endpoint /automation/v1/filters/{id}
      * @param {String} retrieveDir Directory where retrieved metadata directory will be saved
-     * @returns {Promise} Promise
+     * @returns {Promise<{metadata:FilterMap,type:string}>} Promise of items
      */
     static async retrieve(retrieveDir) {
         return super.retrieveREST(retrieveDir, '/automation/v1/filters/', null);
     }
+    /**
+     * manages post retrieve steps
+     * @param {FilterItem} item a single record
+     * @returns {FilterItem} parsed metadata definition
+     */
+    static postRetrieveTasks(item) {
+        return this.parseMetadata(item);
+    }
+    /**
+     * parses retrieved Metadata before saving
+     * @param {FilterItem} metadata a single record
+     * @returns {FilterItem} parsed metadata definition
+     */
+    static parseMetadata(metadata) {
+        try {
+            // folder
+            metadata.r__folder_Path = Util.getFromCache(
+                this.cache,
+                'folder',
+                metadata.categoryId,
+                'ID',
+                'Path'
+            );
+            delete metadata.categoryId;
+
+            // filterDefinition
+            metadata.r__filterDefinition_CustomerKey = Util.getFromCache(
+                this.cache,
+                'filterDefinition',
+                metadata.filterDefinitionId,
+                'id',
+                'key'
+            );
+            delete metadata.filterDefinitionId;
+
+            // source
+            if (metadata.sourceTypeId === 1) {
+                // list
+            } else if (metadata.sourceTypeId === 2) {
+                // dataExtension
+                metadata.r__source_dataExtension_CustomerKey = Util.getFromCache(
+                    this.cache,
+                    'dataExtension',
+                    metadata.sourceObjectId,
+                    'ObjectID',
+                    'CustomerKey'
+                );
+                delete metadata.sourceObjectId;
+                delete metadata.sourceTypeId;
+            } else {
+                Util.logger.error(
+                    `Filter '${metadata.name}' (${metadata.customerKey}): Unsupported source type ${
+                        metadata.sourceTypeId
+                    }=${dataTypes[metadata.sourceTypeId]}`
+                );
+            }
+
+            // target
+            if (metadata.destinationTypeId === 1) {
+                // list
+            } else if (metadata.destinationTypeId === 2) {
+                // dataExtension
+                metadata.r__destination_dataExtension_CustomerKey = Util.getFromCache(
+                    this.cache,
+                    'dataExtension',
+                    metadata.destinationObjectId,
+                    'ObjectID',
+                    'CustomerKey'
+                );
+                delete metadata.destinationObjectId;
+                delete metadata.destinationTypeId;
+            } else {
+                Util.logger.error(
+                    `Filter '${metadata.name}' (${
+                        metadata.customerKey
+                    }): Unsupported destination type ${metadata.destinationTypeId}=${
+                        dataTypes[metadata.destinationTypeId]
+                    }`
+                );
+            }
+        } catch (ex) {
+            Util.logger.error(`Filter '${metadata.name}' (${metadata.customerKey}): ${ex.message}`);
+        }
+        return metadata;
+    }
+    /**
+     * prepares a record for deployment
+     * @param {FilterItem} metadata a single record
+     * @returns {Promise<FilterItem>} Promise of updated single record
+     */
+    static async preDeployTasks(metadata) {
+        // folder
+        if (metadata.r__folder_Path) {
+            metadata.categoryId = Util.getFromCache(
+                this.cache,
+                'folder',
+                metadata.r__folder_Path,
+                'Path',
+                'ID'
+            );
+            delete metadata.r__folder_Path;
+        }
+
+        // filterDefinition
+        if (metadata.r__filterDefinition_CustomerKey) {
+            metadata.filterDefinitionId = Util.getFromCache(
+                this.cache,
+                'filterDefinition',
+                metadata.r__filterDefinition_CustomerKey,
+                'CustomerKey',
+                'ObjectID'
+            );
+            delete metadata.r__filterDefinition_CustomerKey;
+        }
+
+        // source
+        if (metadata.sourceTypeId === 1) {
+            // list
+        } else if (metadata.r__source_dataExtension_CustomerKey) {
+            // dataExtension
+            metadata.sourceObjectId = Util.getFromCache(
+                this.cache,
+                'dataExtension',
+                metadata.r__source_dataExtension_CustomerKey,
+                'CustomerKey',
+                'ObjectID'
+            );
+            metadata.sourceTypeId = 2;
+            delete metadata.r__source_dataExtension_CustomerKey;
+        } else {
+            // assume the type id is still in the metadata
+            throw new Error(
+                `Filter '${metadata.name}' (${metadata.customerKey}): Unsupported source type ${
+                    metadata.sourceTypeId
+                }=${dataTypes[metadata.sourceTypeId]}`
+            );
+        }
+
+        // target
+        if (metadata.destinationTypeId === 1) {
+            // list
+        } else if (metadata.r__destination_dataExtension_CustomerKey) {
+            // dataExtension
+            metadata.destinationObjectId = Util.getFromCache(
+                this.cache,
+                'dataExtension',
+                metadata.r__destination_dataExtension_CustomerKey,
+                'CustomerKey',
+                'ObjectID'
+            );
+            metadata.destinationTypeId = 2;
+            delete metadata.r__destination_dataExtension_CustomerKey;
+        } else {
+            // assume the type id is still in the metadata
+            throw new Error(
+                `Filter '${metadata.name}' (${
+                    metadata.customerKey
+                }): Unsupported destination type ${metadata.destinationTypeId}=${
+                    dataTypes[metadata.destinationTypeId]
+                }`
+            );
+        }
+
+        return metadata;
+    }
 }
 
 // Assign definition to static attributes
diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js
new file mode 100644
index 000000000..2eeadd855
--- /dev/null
+++ b/lib/metadataTypes/FilterDefinition.js
@@ -0,0 +1,279 @@
+'use strict';
+
+/**
+ * @typedef {Object} FilterDefinitionSOAPItem
+ * @property {string} ObjectID id
+ * @property {string} CustomerKey key
+ * @property {Object} [DataFilter] most relevant part that defines the filter
+ * @property {Object} DataFilter.LeftOperand -
+ * @property {string} DataFilter.LeftOperand.Property -
+ * @property {string} DataFilter.LeftOperand.SimpleOperator -
+ * @property {string} DataFilter.LeftOperand.Value -
+ * @property {string} DataFilter.LogicalOperator -
+ * @property {Object} [DataFilter.RightOperand] -
+ * @property {string} DataFilter.RightOperand.Property -
+ * @property {string} DataFilter.RightOperand.SimpleOperator -
+ * @property {string} DataFilter.RightOperand.Value -
+ * @property {string} Name name
+ * @property {string} Description -
+ * @property {string} [ObjectState] returned from SOAP API; used to return error messages
+ *
+ * @typedef {Object.<string, FilterDefinitionSOAPItem>} FilterDefinitionSOAPItemMap
+
+ *
+ * /automation/v1/filterdefinitions/<id> (not used)
+ * @typedef {Object} AutomationFilterDefinitionItem
+ * @property {string} id object id
+ * @property {string} key external key
+ * @property {string} createdDate -
+ * @property {number} createdBy user id
+ * @property {string} createdName -
+ * @property {string} [description] (omitted by API if empty)
+ * @property {string} modifiedDate -
+ * @property {number} modifiedBy user id
+ * @property {string} modifiedName -
+ * @property {string} name name
+ * @property {string} categoryId folder id
+ * @property {string} filterDefinitionXml from REST API defines the filter in XML form
+ * @property {1|2} derivedFromType 1:list/profile attributes/measures, 2: dataExtension
+ * @property {boolean} isSendable ?
+ * @property {Object} [soap__DataFilter] copied from SOAP API, defines the filter in readable form
+ * @property {Object} soap__DataFilter.LeftOperand -
+ * @property {string} soap__DataFilter.LeftOperand.Property -
+ * @property {string} soap__DataFilter.LeftOperand.SimpleOperator -
+ * @property {string} soap__DataFilter.LeftOperand.Value -
+ * @property {string} soap__DataFilter.LogicalOperator -
+ * @property {Object} [soap__DataFilter.RightOperand] -
+ * @property {string} soap__DataFilter.RightOperand.Property -
+ * @property {string} soap__DataFilter.RightOperand.SimpleOperator -
+ * @property {string} soap__DataFilter.RightOperand.Value -
+ *
+ * /email/v1/filters/filterdefinition/<id>
+ * @typedef {Object} FilterDefinitionItem
+ * @property {string} id object id
+ * @property {string} key external key
+ * @property {string} createdDate date
+ * @property {number} createdBy user id
+ * @property {string} createdName name
+ * @property {string} [description] (omitted by API if empty)
+ * @property {string} lastUpdated date
+ * @property {number} lastUpdatedBy user id
+ * @property {string} lastUpdatedName name
+ * @property {string} name name
+ * @property {string} categoryId folder id
+ * @property {string} filterDefinitionXml from REST API defines the filter in XML form
+ * @property {1|2} derivedFromType 1:list/profile attributes/measures, 2: dataExtension
+ * @property {string} derivedFromObjectId Id of DataExtension - present if derivedFromType=2
+ * @property {'DataExtension'|'SubscriberAttributes'} derivedFromObjectTypeName -
+ * @property {string} [derivedFromObjectName] name of DataExtension
+ * @property {boolean} isSendable ?
+ * @property {Object} [soap__DataFilter] copied from SOAP API, defines the filter in readable form
+ * @property {Object} soap__DataFilter.LeftOperand -
+ * @property {string} soap__DataFilter.LeftOperand.Property -
+ * @property {string} soap__DataFilter.LeftOperand.SimpleOperator -
+ * @property {string} soap__DataFilter.LeftOperand.Value -
+ * @property {string} soap__DataFilter.LogicalOperator -
+ * @property {Object} [soap__DataFilter.RightOperand] -
+ * @property {string} soap__DataFilter.RightOperand.Property -
+ * @property {string} soap__DataFilter.RightOperand.SimpleOperator -
+ * @property {string} soap__DataFilter.RightOperand.Value -
+
+ *
+ * @typedef {Object.<string, FilterDefinitionItem>} FilterDefinitionMap
+ */
+
+const MetadataType = require('./MetadataType');
+const Util = require('../util/util');
+const xml2js = require('xml2js');
+
+/**
+ * FilterDefinition MetadataType
+ * @augments MetadataType
+ */
+class FilterDefinition extends MetadataType {
+    /**
+     * Retrieves all records and saves it to disk
+     * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
+     * @returns {Promise<{metadata:FilterDefinitionMap,type:string}>} Promise of items
+     */
+    static async retrieve(retrieveDir) {
+        // #1 get the list via SOAP cause the corresponding REST call has no BU filter apparently
+        // for reference the rest path: '/automation/v1/filterdefinitions?view=categoryinfo'
+        const keyFieldBak = this.definition.keyField;
+        const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Description', 'Name'];
+        this.definition.keyField = 'CustomerKey';
+        /**
+         * @type {FilterDefinitionSOAPItemMap[]}
+         */
+        const responseObject = await this.retrieveSOAPBody(soapFields);
+        this.definition.keyField = keyFieldBak;
+
+        // convert back to array
+        /**
+         * @type {FilterDefinitionSOAPItem[]}
+         */
+        const listResponse = Object.keys(responseObject)
+            .map((key) => responseObject[key])
+            .filter((item) => {
+                if (item.ObjectState) {
+                    Util.logger.debug(
+                        `Filtered filterDefinition ${item.name}: ${item.ObjectState}`
+                    );
+                    return false;
+                } else {
+                    return true;
+                }
+            });
+
+        // #2
+        // /automation/v1/filterdefinitions/<id>
+        const response = (
+            await Promise.all(
+                listResponse.map((item) =>
+                    this.client.RestClient.get({
+                        uri: '/email/v1/filters/filterdefinition/' + item.ObjectID,
+                    })
+                )
+            )
+        )
+            .map((item) => item.body)
+            .map((item) => {
+                // description is not returned when empty
+                item.description = item.description || '';
+                // add extra info from XML
+                item.c__soap_DataFilter = responseObject[item.key].DataFilter;
+                return item;
+            });
+        const results = this.parseResponseBody({ Results: response });
+        if (retrieveDir) {
+            const savedMetadata = await this.saveResults(results, retrieveDir, null, null);
+            Util.logger.info(
+                `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})`
+            );
+        }
+
+        return { metadata: results, type: this.definition.type };
+
+        // return super.retrieveSOAPgeneric(retrieveDir);
+    }
+    /**
+     * Retrieves all records for caching
+     * @returns {Promise<{metadata:FilterDefinitionMap,type:string}>} Promise of items
+     */
+    static async retrieveForCache() {
+        return this.retrieve(null);
+    }
+
+    /**
+     * manages post retrieve steps
+     * @param {FilterDefinitionItem} item a single record
+     * @returns {FilterDefinitionItem} parsed metadata definition
+     */
+    static async postRetrieveTasks(item) {
+        return this.parseMetadata(item);
+    }
+    /**
+     * parses retrieved Metadata before saving
+     * @param {FilterDefinitionItem} metadata a single record
+     * @returns {FilterDefinitionItem} parsed metadata definition
+     */
+    static async parseMetadata(metadata) {
+        try {
+            // folder
+            metadata.r__folder_Path = Util.getFromCache(
+                this.cache,
+                'folder',
+                metadata.categoryId,
+                'ID',
+                'Path'
+            );
+            delete metadata.categoryId;
+
+            if (metadata.derivedFromType === 2) {
+                // DataExtension
+                metadata.r__dataExtension_CustomerKey = Util.getFromCache(
+                    this.cache,
+                    'dataExtension',
+                    metadata.derivedFromObjectId,
+                    'ObjectID',
+                    'CustomerKey'
+                );
+            }
+            delete metadata.derivedFromObjectId;
+            delete metadata.derivedFromType;
+            metadata.c__filterDefinition = await xml2js.parseStringPromise(
+                metadata.filterDefinitionXml /* , options */
+            );
+
+            // TODO check if Condition ID needs to be resolved or can be ignored
+        } catch (ex) {
+            Util.logger.error(
+                `FilterDefinition '${metadata.name}' (${metadata.key}): ${ex.message}`
+            );
+        }
+        return metadata;
+    }
+    /**
+     * prepares a item for deployment
+     * @param {FilterDefinitionItem} metadata a single record
+     * @returns {Promise<FilterDefinitionItem>} Promise of updated single item
+     */
+    static async preDeployTasks(metadata) {
+        // folder
+        metadata.categoryId = Util.getFromCache(
+            this.cache,
+            'folder',
+            metadata.r__folder_Path,
+            'Path',
+            'ID'
+        );
+        delete metadata.r__folder_Path;
+
+        if (metadata.derivedFromObjectTypeName === 'SubscriberAttributes') {
+            // List
+            metadata.derivedFromType = 1;
+            metadata.derivedFromObjectId = '00000000-0000-0000-0000-000000000000';
+        } else {
+            // DataExtension
+            metadata.derivedFromType = 2;
+
+            if (metadata.r__dataExtension_CustomerKey) {
+                metadata.derivedFromObjectId = Util.getFromCache(
+                    this.cache,
+                    'dataExtension',
+                    metadata.r__dataExtension_CustomerKey,
+                    'CustomerKey',
+                    'ObjectID'
+                );
+                delete metadata.r__dataExtension_CustomerKey;
+            }
+        }
+        delete metadata.c__filterDefinition;
+        delete metadata.c__soap_DataFilter;
+
+        return metadata;
+    }
+    /**
+     * Creates a single item
+     * @param {FilterDefinitionItem} metadata a single item
+     * @returns {Promise<FilterDefinitionItem>} Promise
+     */
+    static create(metadata) {
+        // TODO test the create
+        return super.createREST(metadata, '/email/v1/filters/filterdefinition/');
+    }
+    /**
+     * Updates a single item
+     * @param {FilterDefinitionItem} metadata a single item
+     * @returns {Promise<FilterDefinitionItem>} Promise
+     */
+    static update(metadata) {
+        // TODO test the update
+        // TODO figure out how to get the ID on the fly
+        return super.updateREST(metadata, '/email/v1/filters/filterdefinition/' + metadata.Id);
+    }
+}
+// Assign definition to static attributes
+FilterDefinition.definition = require('../MetadataTypeDefinitions').filterDefinition;
+
+module.exports = FilterDefinition;
diff --git a/lib/metadataTypes/definitions/Filter.definition.js b/lib/metadataTypes/definitions/Filter.definition.js
index b9503571f..8f1686f6c 100644
--- a/lib/metadataTypes/definitions/Filter.definition.js
+++ b/lib/metadataTypes/definitions/Filter.definition.js
@@ -1,6 +1,6 @@
 module.exports = {
     bodyIteratorField: 'items',
-    dependencies: [],
+    dependencies: ['filterDefinition', 'list', 'dataExtension', 'folder'],
     hasExtended: false,
     idField: 'id',
     keyField: 'customerKey',
@@ -8,56 +8,58 @@ module.exports = {
     restPagination: true,
     type: 'filter',
     typeDescription:
-        'BETA: Part of how filtered Data Extensions are created. Depends on type "FilterDefinitions".',
-    typeRetrieveByDefault: false,
+        'Used in automations to filter lists and DEs. Depends on type "FilterDefinitions".',
+    typeRetrieveByDefault: true,
     typeName: 'Automation: Filter Activity',
     fields: {
+        // https://developer.salesforce.com/docs/atlas.en-us.noversion.mc-apis.meta/mc-apis/filteractivity.htm
         categoryId: {
-            isCreateable: false,
-            isUpdateable: false,
+            isCreateable: true,
+            isUpdateable: true,
             retrieving: true,
-            template: false,
+            template: true,
         },
         createdDate: {
             isCreateable: false,
             isUpdateable: false,
-            retrieving: true,
+            retrieving: false,
             template: false,
         },
         customerKey: {
-            isCreateable: null,
-            isUpdateable: false,
+            isCreateable: true,
+            isUpdateable: true,
             retrieving: true,
-            template: false,
+            template: true,
         },
         description: {
-            isCreateable: false,
-            isUpdateable: false,
+            isCreateable: true,
+            isUpdateable: true,
             retrieving: true,
-            template: false,
+            template: true,
         },
         destinationObjectId: {
-            isCreateable: false,
-            isUpdateable: false,
+            isCreateable: true,
+            isUpdateable: true,
             retrieving: true,
-            template: false,
+            template: true,
         },
         destinationTypeId: {
-            isCreateable: false,
-            isUpdateable: false,
+            isCreateable: true,
+            isUpdateable: true,
             retrieving: true,
-            template: false,
+            template: true,
         },
         filterActivityId: {
             isCreateable: null,
             isUpdateable: null,
             retrieving: true,
+            template: true,
         },
         filterDefinitionId: {
-            isCreateable: false,
-            isUpdateable: false,
+            isCreateable: true,
+            isUpdateable: true,
             retrieving: true,
-            template: false,
+            template: true,
         },
         modifiedDate: {
             isCreateable: false,
@@ -66,28 +68,28 @@ module.exports = {
             template: false,
         },
         name: {
-            isCreateable: null,
-            isUpdateable: false,
+            isCreateable: true,
+            isUpdateable: true,
             retrieving: true,
-            template: false,
+            template: true,
         },
         sourceObjectId: {
-            isCreateable: false,
-            isUpdateable: false,
+            isCreateable: true,
+            isUpdateable: true,
             retrieving: true,
-            template: false,
+            template: true,
         },
         sourceTypeId: {
-            isCreateable: false,
-            isUpdateable: false,
+            isCreateable: true,
+            isUpdateable: true,
             retrieving: true,
-            template: false,
+            template: true,
         },
         statusId: {
-            isCreateable: false,
-            isUpdateable: false,
+            isCreateable: true,
+            isUpdateable: true,
             retrieving: true,
-            template: false,
+            template: true,
         },
     },
 };
diff --git a/lib/metadataTypes/definitions/FilterDefinition.definition.js b/lib/metadataTypes/definitions/FilterDefinition.definition.js
new file mode 100644
index 000000000..77f1c62b7
--- /dev/null
+++ b/lib/metadataTypes/definitions/FilterDefinition.definition.js
@@ -0,0 +1,113 @@
+module.exports = {
+    bodyIteratorField: 'Results',
+    dependencies: ['folder', 'dataExtension'],
+    filter: {},
+    hasExtended: false,
+    idField: 'id',
+    keyField: 'key',
+    nameField: 'name',
+    restPagination: false,
+    type: 'filterDefinition',
+    typeDescription:
+        'Defines an audience based on specified rules. Used by Filter Activities and Filtered DEs.',
+    typeRetrieveByDefault: true,
+    typeName: 'Filter Definition',
+    fields: {
+        id: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        key: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        createdDate: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        createdBy: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        createdByName: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        lastUpdated: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true,
+            template: false,
+        },
+        lastUpdatedBy: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        lastUpdatedName: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        name: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        categoryId: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        filterDefinitionXml: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        derivedFromType: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        derivedFromObjectId: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        derivedFromObjectTypeName: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        derivedFromObjectName: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true,
+            template: true,
+        },
+        isSendable: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+    },
+};

From d4a0e53cade7f52e09008cbcd708a785c2d35fa2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= <joern.berkefeld@accenture.com>
Date: Sun, 17 Apr 2022 09:58:52 +0200
Subject: [PATCH 2/8] #9: make branch ready for 4.0.0 release

---
 lib/metadataTypes/Filter.js           | 31 ++++-----
 lib/metadataTypes/FilterDefinition.js | 94 +++++++++++++--------------
 package-lock.json                     |  1 +
 package.json                          |  1 +
 4 files changed, 58 insertions(+), 69 deletions(-)

diff --git a/lib/metadataTypes/Filter.js b/lib/metadataTypes/Filter.js
index b45000ff4..c26e25a19 100644
--- a/lib/metadataTypes/Filter.js
+++ b/lib/metadataTypes/Filter.js
@@ -20,6 +20,7 @@
 
 const MetadataType = require('./MetadataType');
 const Util = require('../util/util');
+const cache = require('../util/cache');
 
 const dataTypes = {
     1: 'List',
@@ -60,8 +61,7 @@ class Filter extends MetadataType {
     static parseMetadata(metadata) {
         try {
             // folder
-            metadata.r__folder_Path = Util.getFromCache(
-                this.cache,
+            metadata.r__folder_Path = cache.searchForField(
                 'folder',
                 metadata.categoryId,
                 'ID',
@@ -70,8 +70,7 @@ class Filter extends MetadataType {
             delete metadata.categoryId;
 
             // filterDefinition
-            metadata.r__filterDefinition_CustomerKey = Util.getFromCache(
-                this.cache,
+            metadata.r__filterDefinition_CustomerKey = cache.searchForField(
                 'filterDefinition',
                 metadata.filterDefinitionId,
                 'id',
@@ -84,8 +83,7 @@ class Filter extends MetadataType {
                 // list
             } else if (metadata.sourceTypeId === 2) {
                 // dataExtension
-                metadata.r__source_dataExtension_CustomerKey = Util.getFromCache(
-                    this.cache,
+                metadata.r__source_dataExtension_CustomerKey = cache.searchForField(
                     'dataExtension',
                     metadata.sourceObjectId,
                     'ObjectID',
@@ -94,7 +92,7 @@ class Filter extends MetadataType {
                 delete metadata.sourceObjectId;
                 delete metadata.sourceTypeId;
             } else {
-                Util.logger.error(
+                Util.logger.warn(
                     `Filter '${metadata.name}' (${metadata.customerKey}): Unsupported source type ${
                         metadata.sourceTypeId
                     }=${dataTypes[metadata.sourceTypeId]}`
@@ -106,8 +104,7 @@ class Filter extends MetadataType {
                 // list
             } else if (metadata.destinationTypeId === 2) {
                 // dataExtension
-                metadata.r__destination_dataExtension_CustomerKey = Util.getFromCache(
-                    this.cache,
+                metadata.r__destination_dataExtension_CustomerKey = cache.searchForField(
                     'dataExtension',
                     metadata.destinationObjectId,
                     'ObjectID',
@@ -116,7 +113,7 @@ class Filter extends MetadataType {
                 delete metadata.destinationObjectId;
                 delete metadata.destinationTypeId;
             } else {
-                Util.logger.error(
+                Util.logger.warn(
                     `Filter '${metadata.name}' (${
                         metadata.customerKey
                     }): Unsupported destination type ${metadata.destinationTypeId}=${
@@ -125,7 +122,7 @@ class Filter extends MetadataType {
                 );
             }
         } catch (ex) {
-            Util.logger.error(`Filter '${metadata.name}' (${metadata.customerKey}): ${ex.message}`);
+            Util.logger.warn(`Filter '${metadata.name}' (${metadata.customerKey}): ${ex.message}`);
         }
         return metadata;
     }
@@ -137,8 +134,7 @@ class Filter extends MetadataType {
     static async preDeployTasks(metadata) {
         // folder
         if (metadata.r__folder_Path) {
-            metadata.categoryId = Util.getFromCache(
-                this.cache,
+            metadata.categoryId = cache.searchForField(
                 'folder',
                 metadata.r__folder_Path,
                 'Path',
@@ -149,8 +145,7 @@ class Filter extends MetadataType {
 
         // filterDefinition
         if (metadata.r__filterDefinition_CustomerKey) {
-            metadata.filterDefinitionId = Util.getFromCache(
-                this.cache,
+            metadata.filterDefinitionId = cache.searchForField(
                 'filterDefinition',
                 metadata.r__filterDefinition_CustomerKey,
                 'CustomerKey',
@@ -164,8 +159,7 @@ class Filter extends MetadataType {
             // list
         } else if (metadata.r__source_dataExtension_CustomerKey) {
             // dataExtension
-            metadata.sourceObjectId = Util.getFromCache(
-                this.cache,
+            metadata.sourceObjectId = cache.searchForField(
                 'dataExtension',
                 metadata.r__source_dataExtension_CustomerKey,
                 'CustomerKey',
@@ -187,8 +181,7 @@ class Filter extends MetadataType {
             // list
         } else if (metadata.r__destination_dataExtension_CustomerKey) {
             // dataExtension
-            metadata.destinationObjectId = Util.getFromCache(
-                this.cache,
+            metadata.destinationObjectId = cache.searchForField(
                 'dataExtension',
                 metadata.r__destination_dataExtension_CustomerKey,
                 'CustomerKey',
diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js
index 2eeadd855..51abe9516 100644
--- a/lib/metadataTypes/FilterDefinition.js
+++ b/lib/metadataTypes/FilterDefinition.js
@@ -84,7 +84,8 @@
 
 const MetadataType = require('./MetadataType');
 const Util = require('../util/util');
-const xml2js = require('xml2js');
+const cache = require('../util/cache');
+const { XMLBuilder, XMLParser } = require('fast-xml-parser');
 
 /**
  * FilterDefinition MetadataType
@@ -99,52 +100,51 @@ class FilterDefinition extends MetadataType {
     static async retrieve(retrieveDir) {
         // #1 get the list via SOAP cause the corresponding REST call has no BU filter apparently
         // for reference the rest path: '/automation/v1/filterdefinitions?view=categoryinfo'
-        const keyFieldBak = this.definition.keyField;
+
         const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Description', 'Name'];
-        this.definition.keyField = 'CustomerKey';
         /**
          * @type {FilterDefinitionSOAPItemMap[]}
          */
-        const responseObject = await this.retrieveSOAPBody(soapFields);
+        const responseSOAP = await this.client.soap.retrieveBulk(this.definition.type, soapFields);
+        console.log('responseSOAP', responseSOAP);
+
+        // backup REST value of the keyField
+        const keyFieldBak = this.definition.keyField;
+        this.definition.keyField = 'CustomerKey';
+        const responseSOAPMap = this.parseResponseBody(responseSOAP);
+        // restore the keyField to its REST value
         this.definition.keyField = keyFieldBak;
+        console.log('responseSOAPMap', responseSOAPMap);
 
-        // convert back to array
         /**
          * @type {FilterDefinitionSOAPItem[]}
          */
-        const listResponse = Object.keys(responseObject)
-            .map((key) => responseObject[key])
-            .filter((item) => {
-                if (item.ObjectState) {
-                    Util.logger.debug(
-                        `Filtered filterDefinition ${item.name}: ${item.ObjectState}`
-                    );
-                    return false;
-                } else {
-                    return true;
-                }
-            });
+        const responseSOAPList = responseSOAP.Results.filter((item) => {
+            if (item.ObjectState) {
+                Util.logger.debug(`Filtered filterDefinition ${item.name}: ${item.ObjectState}`);
+                return false;
+            } else {
+                return true;
+            }
+        });
 
         // #2
         // /automation/v1/filterdefinitions/<id>
-        const response = (
+        const responseREST = (
             await Promise.all(
-                listResponse.map((item) =>
-                    this.client.RestClient.get({
-                        uri: '/email/v1/filters/filterdefinition/' + item.ObjectID,
-                    })
+                responseSOAPList.map((item) =>
+                    this.client.rest.get('/email/v1/filters/filterdefinition/' + item.ObjectID)
                 )
             )
-        )
-            .map((item) => item.body)
-            .map((item) => {
-                // description is not returned when empty
-                item.description = item.description || '';
-                // add extra info from XML
-                item.c__soap_DataFilter = responseObject[item.key].DataFilter;
-                return item;
-            });
-        const results = this.parseResponseBody({ Results: response });
+        ).map((item) => {
+            // description is not returned when empty
+            item.description = item.description || '';
+            // add extra info from XML
+            item.c__soap_DataFilter = responseSOAPMap[item.key].DataFilter;
+            return item;
+        });
+        console.log('responseREST', responseREST);
+        const results = this.parseResponseBody({ Results: responseREST });
         if (retrieveDir) {
             const savedMetadata = await this.saveResults(results, retrieveDir, null, null);
             Util.logger.info(
@@ -153,8 +153,6 @@ class FilterDefinition extends MetadataType {
         }
 
         return { metadata: results, type: this.definition.type };
-
-        // return super.retrieveSOAPgeneric(retrieveDir);
     }
     /**
      * Retrieves all records for caching
@@ -180,8 +178,7 @@ class FilterDefinition extends MetadataType {
     static async parseMetadata(metadata) {
         try {
             // folder
-            metadata.r__folder_Path = Util.getFromCache(
-                this.cache,
+            metadata.r__folder_Path = cache.searchForField(
                 'folder',
                 metadata.categoryId,
                 'ID',
@@ -191,19 +188,20 @@ class FilterDefinition extends MetadataType {
 
             if (metadata.derivedFromType === 2) {
                 // DataExtension
-                metadata.r__dataExtension_CustomerKey = Util.getFromCache(
-                    this.cache,
+                metadata.r__dataExtension_CustomerKey = cache.searchForField(
                     'dataExtension',
                     metadata.derivedFromObjectId,
                     'ObjectID',
                     'CustomerKey'
                 );
             }
+            metadata.del__derivedFromObjectId = metadata.derivedFromObjectId; // TEMP for DEBUGGING / remove before release
+            metadata.del__derivedFromType = metadata.derivedFromType; // TEMP for DEBUGGING / remove before release
             delete metadata.derivedFromObjectId;
             delete metadata.derivedFromType;
-            metadata.c__filterDefinition = await xml2js.parseStringPromise(
-                metadata.filterDefinitionXml /* , options */
-            );
+
+            const xmlToJson = new XMLParser({ ignoreAttributes: false });
+            metadata.c__filterDefinition = xmlToJson.parse(metadata.filterDefinitionXml);
 
             // TODO check if Condition ID needs to be resolved or can be ignored
         } catch (ex) {
@@ -220,13 +218,7 @@ class FilterDefinition extends MetadataType {
      */
     static async preDeployTasks(metadata) {
         // folder
-        metadata.categoryId = Util.getFromCache(
-            this.cache,
-            'folder',
-            metadata.r__folder_Path,
-            'Path',
-            'ID'
-        );
+        metadata.categoryId = cache.searchForField('folder', metadata.r__folder_Path, 'Path', 'ID');
         delete metadata.r__folder_Path;
 
         if (metadata.derivedFromObjectTypeName === 'SubscriberAttributes') {
@@ -238,8 +230,7 @@ class FilterDefinition extends MetadataType {
             metadata.derivedFromType = 2;
 
             if (metadata.r__dataExtension_CustomerKey) {
-                metadata.derivedFromObjectId = Util.getFromCache(
-                    this.cache,
+                metadata.derivedFromObjectId = cache.searchForField(
                     'dataExtension',
                     metadata.r__dataExtension_CustomerKey,
                     'CustomerKey',
@@ -248,6 +239,9 @@ class FilterDefinition extends MetadataType {
                 delete metadata.r__dataExtension_CustomerKey;
             }
         }
+
+        const jsonToXml = new XMLBuilder({ ignoreAttributes: false });
+        metadata.filterDefinitionXml = jsonToXml.build(metadata.c__filterDefinition);
         delete metadata.c__filterDefinition;
         delete metadata.c__soap_DataFilter;
 
diff --git a/package-lock.json b/package-lock.json
index b7d320163..5bfecead4 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -14,6 +14,7 @@
                 "command-exists": "1.2.9",
                 "conf": "10.1.1",
                 "console.table": "0.10.0",
+                "fast-xml-parser": "4.0.7",
                 "fs-extra": "10.0.1",
                 "inquirer": "8.2.2",
                 "json-to-table": "4.2.1",
diff --git a/package.json b/package.json
index 9cce2236d..5f57cba68 100644
--- a/package.json
+++ b/package.json
@@ -41,6 +41,7 @@
         "command-exists": "1.2.9",
         "conf": "10.1.1",
         "console.table": "0.10.0",
+        "fast-xml-parser": "4.0.7",
         "fs-extra": "10.0.1",
         "inquirer": "8.2.2",
         "json-to-table": "4.2.1",

From 0e519dba94f846fb624d4d4cbcc4d218511d4aef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= <joern.berkefeld@accenture.com>
Date: Tue, 29 Aug 2023 10:14:06 +0200
Subject: [PATCH 3/8] #9: ran lint:fix

---
 lib/metadataTypes/FilterDefinition.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js
index 12840fe24..979a96baa 100644
--- a/lib/metadataTypes/FilterDefinition.js
+++ b/lib/metadataTypes/FilterDefinition.js
@@ -59,7 +59,7 @@ class FilterDefinition extends MetadataType {
             )
         ).map((item) => {
             // description is not returned when empty
-            item.description = item.description || '';
+            item.description ||= '';
             // add extra info from XML
             item.c__soap_DataFilter = responseSOAPMap[item.key].DataFilter;
             return item;

From a742713496a8e6858fb12284608737a2748fc5b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= <joern.berkefeld@accenture.com>
Date: Tue, 29 Aug 2023 15:08:18 +0200
Subject: [PATCH 4/8] #9: improve retrieve filterDefinition

---
 docs/dist/documentation.md                    |  24 +--
 lib/metadataTypes/FilterDefinition.js         | 140 +++++++++++-------
 .../FilterDefinition.definition.js            |  72 ++++++---
 3 files changed, 140 insertions(+), 96 deletions(-)

diff --git a/docs/dist/documentation.md b/docs/dist/documentation.md
index 1877f430a..319125d8c 100644
--- a/docs/dist/documentation.md
+++ b/docs/dist/documentation.md
@@ -2951,17 +2951,16 @@ FilterDefinition MetadataType
 **Extends**: [<code>MetadataType</code>](#MetadataType)  
 
 * [FilterDefinition](#FilterDefinition) ⇐ [<code>MetadataType</code>](#MetadataType)
-    * [.retrieve(retrieveDir)](#FilterDefinition.retrieve) ⇒ <code>Promise.&lt;{metadata: TYPE.FilterDefinitionMap, type: string}&gt;</code>
+    * [.retrieve(retrieveDir, [_], [__], [key])](#FilterDefinition.retrieve) ⇒ <code>Promise.&lt;{metadata: TYPE.FilterDefinitionMap, type: string}&gt;</code>
     * [.retrieveForCache()](#FilterDefinition.retrieveForCache) ⇒ <code>Promise.&lt;{metadata: TYPE.FilterDefinitionMap, type: string}&gt;</code>
-    * [.postRetrieveTasks(item)](#FilterDefinition.postRetrieveTasks) ⇒ <code>TYPE.FilterDefinitionItem</code>
-    * [.parseMetadata(metadata)](#FilterDefinition.parseMetadata) ⇒ <code>TYPE.FilterDefinitionItem</code>
+    * [.postRetrieveTasks(metadata)](#FilterDefinition.postRetrieveTasks) ⇒ <code>TYPE.FilterDefinitionItem</code>
     * [.preDeployTasks(metadata)](#FilterDefinition.preDeployTasks) ⇒ <code>Promise.&lt;TYPE.FilterDefinitionItem&gt;</code>
     * [.create(metadata)](#FilterDefinition.create) ⇒ <code>Promise.&lt;TYPE.FilterDefinitionItem&gt;</code>
     * [.update(metadata)](#FilterDefinition.update) ⇒ <code>Promise.&lt;TYPE.FilterDefinitionItem&gt;</code>
 
 <a name="FilterDefinition.retrieve"></a>
 
-### FilterDefinition.retrieve(retrieveDir) ⇒ <code>Promise.&lt;{metadata: TYPE.FilterDefinitionMap, type: string}&gt;</code>
+### FilterDefinition.retrieve(retrieveDir, [_], [__], [key]) ⇒ <code>Promise.&lt;{metadata: TYPE.FilterDefinitionMap, type: string}&gt;</code>
 Retrieves all records and saves it to disk
 
 **Kind**: static method of [<code>FilterDefinition</code>](#FilterDefinition)  
@@ -2970,6 +2969,9 @@ Retrieves all records and saves it to disk
 | Param | Type | Description |
 | --- | --- | --- |
 | retrieveDir | <code>string</code> | Directory where retrieved metadata directory will be saved |
+| [_] | <code>void</code> | unused parameter |
+| [__] | <code>void</code> | unused parameter |
+| [key] | <code>string</code> | customer key of single item to retrieve |
 
 <a name="FilterDefinition.retrieveForCache"></a>
 
@@ -2980,19 +2982,7 @@ Retrieves all records for caching
 **Returns**: <code>Promise.&lt;{metadata: TYPE.FilterDefinitionMap, type: string}&gt;</code> - Promise of items  
 <a name="FilterDefinition.postRetrieveTasks"></a>
 
-### FilterDefinition.postRetrieveTasks(item) ⇒ <code>TYPE.FilterDefinitionItem</code>
-manages post retrieve steps
-
-**Kind**: static method of [<code>FilterDefinition</code>](#FilterDefinition)  
-**Returns**: <code>TYPE.FilterDefinitionItem</code> - parsed metadata definition  
-
-| Param | Type | Description |
-| --- | --- | --- |
-| item | <code>TYPE.FilterDefinitionItem</code> | a single record |
-
-<a name="FilterDefinition.parseMetadata"></a>
-
-### FilterDefinition.parseMetadata(metadata) ⇒ <code>TYPE.FilterDefinitionItem</code>
+### FilterDefinition.postRetrieveTasks(metadata) ⇒ <code>TYPE.FilterDefinitionItem</code>
 parses retrieved Metadata before saving
 
 **Kind**: static method of [<code>FilterDefinition</code>](#FilterDefinition)  
diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js
index 979a96baa..5422e72eb 100644
--- a/lib/metadataTypes/FilterDefinition.js
+++ b/lib/metadataTypes/FilterDefinition.js
@@ -16,33 +16,49 @@ class FilterDefinition extends MetadataType {
      * Retrieves all records and saves it to disk
      *
      * @param {string} retrieveDir Directory where retrieved metadata directory will be saved
+     * @param {void} [_] unused parameter
+     * @param {void} [__] unused parameter
+     * @param {string} [key] customer key of single item to retrieve
      * @returns {Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}>} Promise of items
      */
-    static async retrieve(retrieveDir) {
+    static async retrieve(retrieveDir, _, __, key) {
         // #1 get the list via SOAP cause the corresponding REST call has no BU filter apparently
         // for reference the rest path: '/automation/v1/filterdefinitions?view=categoryinfo'
 
-        const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Description', 'Name'];
+        const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Name'];
+        let requestParams;
+        if (key) {
+            requestParams = {
+                filter: {
+                    leftOperand: 'CustomerKey',
+                    operator: 'equals',
+                    rightOperand: key,
+                },
+            };
+        }
+
         /**
          * @type {TYPE.FilterDefinitionSOAPItemMap[]}
          */
-        const responseSOAP = await this.client.soap.retrieveBulk(this.definition.type, soapFields);
-        console.log('responseSOAP', responseSOAP); // eslint-disable-line no-console
+        const responseSOAP = await this.client.soap.retrieveBulk(
+            this.definition.type,
+            soapFields,
+            requestParams
+        );
 
         // backup REST value of the keyField
         const keyFieldBak = this.definition.keyField;
         this.definition.keyField = 'CustomerKey';
-        const responseSOAPMap = this.parseResponseBody(responseSOAP);
+        const responseSOAPMap = this.parseResponseBody(responseSOAP, key);
         // restore the keyField to its REST value
         this.definition.keyField = keyFieldBak;
-        console.log('responseSOAPMap', responseSOAPMap); // eslint-disable-line no-console
 
         /**
          * @type {TYPE.FilterDefinitionSOAPItem[]}
          */
         const responseSOAPList = responseSOAP.Results.filter((item) => {
             if (item.ObjectState) {
-                Util.logger.debug(`Filtered filterDefinition ${item.name}: ${item.ObjectState}`);
+                Util.logger.debug(`Filtered filterDefinition ${item.Name}: ${item.ObjectState}`);
                 return false;
             } else {
                 return true;
@@ -51,29 +67,28 @@ class FilterDefinition extends MetadataType {
 
         // #2
         // /automation/v1/filterdefinitions/<id>
-        const responseREST = (
-            await Promise.all(
-                responseSOAPList.map((item) =>
-                    this.client.rest.get('/email/v1/filters/filterdefinition/' + item.ObjectID)
-                )
+        const metadataMap = (
+            await super.retrieveRESTcollection(
+                responseSOAPList.map((item) => ({
+                    id: item.ObjectID,
+                    uri: '/email/v1/filters/filterdefinition/' + item.ObjectID,
+                }))
             )
-        ).map((item) => {
+        ).metadata;
+        for (const item of Object.values(metadataMap)) {
             // description is not returned when empty
             item.description ||= '';
             // add extra info from XML
             item.c__soap_DataFilter = responseSOAPMap[item.key].DataFilter;
-            return item;
-        });
-        console.log('responseREST', responseREST); // eslint-disable-line no-console
-        const results = this.parseResponseBody({ Results: responseREST });
+        }
         if (retrieveDir) {
-            const savedMetadata = await this.saveResults(results, retrieveDir, null, null);
+            const savedMetadata = await this.saveResults(metadataMap, retrieveDir);
             Util.logger.info(
                 `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})`
             );
         }
 
-        return { metadata: results, type: this.definition.type };
+        return { metadata: metadataMap, type: this.definition.type };
     }
     /**
      * Retrieves all records for caching
@@ -84,50 +99,62 @@ class FilterDefinition extends MetadataType {
         return this.retrieve(null);
     }
 
-    /**
-     * manages post retrieve steps
-     *
-     * @param {TYPE.FilterDefinitionItem} item a single record
-     * @returns {TYPE.FilterDefinitionItem} parsed metadata definition
-     */
-    static async postRetrieveTasks(item) {
-        return this.parseMetadata(item);
-    }
     /**
      * parses retrieved Metadata before saving
      *
      * @param {TYPE.FilterDefinitionItem} metadata a single record
      * @returns {TYPE.FilterDefinitionItem} parsed metadata definition
      */
-    static async parseMetadata(metadata) {
+    static async postRetrieveTasks(metadata) {
+        if (metadata.derivedFromType > 4) {
+            // GUI only shows types 1,2,3,4; lets mimic that here.
+            // type 6 seems to be journey related. Maybe we need to change that again in the future
+            return;
+        }
         try {
             // folder
-            metadata.r__folder_Path = cache.searchForField(
-                'folder',
-                metadata.categoryId,
-                'ID',
-                'Path'
-            );
-            delete metadata.categoryId;
-
-            if (metadata.derivedFromType === 2) {
-                // DataExtension
-                metadata.r__dataExtension_CustomerKey = cache.searchForField(
-                    'dataExtension',
-                    metadata.derivedFromObjectId,
-                    'ObjectID',
-                    'CustomerKey'
-                );
+            this.setFolderPath(metadata);
+
+            switch (metadata.derivedFromType) {
+                case 1: {
+                    // SubscriberAttributes
+                    // TODO
+                    break;
+                }
+                case 2: {
+                    // DataExtension
+                    metadata.r__dataExtension_CustomerKey = cache.searchForField(
+                        'dataExtension',
+                        metadata.derivedFromObjectId,
+                        'ObjectID',
+                        'CustomerKey'
+                    );
+                    delete metadata.derivedFromObjectId;
+                    delete metadata.derivedFromType;
+                    break;
+                }
+                case 3: {
+                    // TODO
+                    break;
+                }
+                case 4: {
+                    // TODO
+                    break;
+                }
+                case 5: {
+                    // TODO
+                    break;
+                }
+                case 6: {
+                    // TODO
+                    break;
+                }
             }
-            metadata.del__derivedFromObjectId = metadata.derivedFromObjectId; // TEMP for DEBUGGING / remove before release
-            metadata.del__derivedFromType = metadata.derivedFromType; // TEMP for DEBUGGING / remove before release
-            delete metadata.derivedFromObjectId;
-            delete metadata.derivedFromType;
 
             const xmlToJson = new XMLParser({ ignoreAttributes: false });
             metadata.c__filterDefinition = xmlToJson.parse(metadata.filterDefinitionXml);
-
-            // TODO check if Condition ID needs to be resolved or can be ignored
+            // TODO map Condition ID to DataExtensionField ID
+            delete metadata.filterDefinitionXml;
         } catch (ex) {
             Util.logger.error(
                 `FilterDefinition '${metadata.name}' (${metadata.key}): ${ex.message}`
@@ -143,11 +170,10 @@ class FilterDefinition extends MetadataType {
      */
     static async preDeployTasks(metadata) {
         // folder
-        metadata.categoryId = cache.searchForField('folder', metadata.r__folder_Path, 'Path', 'ID');
-        delete metadata.r__folder_Path;
+        super.setFolderId(metadata);
 
         if (metadata.derivedFromObjectTypeName === 'SubscriberAttributes') {
-            // List
+            // SubscriberAttributes
             metadata.derivedFromType = 1;
             metadata.derivedFromObjectId = '00000000-0000-0000-0000-000000000000';
         } else {
@@ -190,8 +216,10 @@ class FilterDefinition extends MetadataType {
      */
     static update(metadata) {
         // TODO test the update
-        // TODO figure out how to get the ID on the fly
-        return super.updateREST(metadata, '/email/v1/filters/filterdefinition/' + metadata.Id);
+        return super.updateREST(
+            metadata,
+            '/email/v1/filters/filterdefinition/' + metadata[this.definition.idField]
+        );
     }
 }
 // Assign definition to static attributes
diff --git a/lib/metadataTypes/definitions/FilterDefinition.definition.js b/lib/metadataTypes/definitions/FilterDefinition.definition.js
index 77f1c62b7..83f51d78a 100644
--- a/lib/metadataTypes/definitions/FilterDefinition.definition.js
+++ b/lib/metadataTypes/definitions/FilterDefinition.definition.js
@@ -1,11 +1,17 @@
 module.exports = {
     bodyIteratorField: 'Results',
-    dependencies: ['folder', 'dataExtension'],
+    dependencies: ['folder-filterdefinition', 'folder-hidden', 'dataExtension'],
     filter: {},
     hasExtended: false,
     idField: 'id',
     keyField: 'key',
     nameField: 'name',
+    folderType: 'filterdefinition',
+    folderIdField: 'categoryId',
+    createdDateField: 'createdDate',
+    createdNameField: 'createdBy',
+    lastmodDateField: 'lastUpdated',
+    lastmodNameField: 'lastUpdatedBy',
     restPagination: false,
     type: 'filterDefinition',
     typeDescription:
@@ -13,9 +19,10 @@ module.exports = {
     typeRetrieveByDefault: true,
     typeName: 'Filter Definition',
     fields: {
+        // the GUI seems to ONLY send fields during update that are actually changed. It has yet to be tested if that also works with sending other fiedls as well
         id: {
             isCreateable: false,
-            isUpdateable: false,
+            isUpdateable: false, // included in URL
             retrieving: false,
             template: false,
         },
@@ -37,30 +44,30 @@ module.exports = {
             retrieving: false,
             template: false,
         },
-        createdByName: {
-            isCreateable: false,
-            isUpdateable: false,
-            retrieving: false,
-            template: false,
-        },
+        // createdByName: {
+        //     isCreateable: false,
+        //     isUpdateable: false,
+        //     retrieving: false,
+        //     template: false,
+        // },
         lastUpdated: {
-            isCreateable: false,
-            isUpdateable: false,
-            retrieving: true,
-            template: false,
-        },
-        lastUpdatedBy: {
             isCreateable: false,
             isUpdateable: false,
             retrieving: false,
             template: false,
         },
-        lastUpdatedName: {
+        lastUpdatedBy: {
             isCreateable: false,
             isUpdateable: false,
             retrieving: false,
             template: false,
         },
+        // lastUpdatedName: {
+        //     isCreateable: false,
+        //     isUpdateable: false,
+        //     retrieving: false,
+        //     template: false,
+        // },
         name: {
             isCreateable: true,
             isUpdateable: true,
@@ -68,44 +75,63 @@ module.exports = {
             template: true,
         },
         categoryId: {
+            // returned by GET / CREATE / UPDATE; used in CREATE payload
             isCreateable: true,
             isUpdateable: true,
             retrieving: true,
             template: true,
         },
+        // CategoryId: {
+        //     // used by UPDATE payload
+        //     isCreateable: false,
+        //     isUpdateable: true,
+        //     retrieving: false,
+        //     template: false,
+        // },
         filterDefinitionXml: {
             isCreateable: true,
             isUpdateable: true,
             retrieving: true,
             template: true,
         },
+        // DerivedFromType: {
+        //     // this upper-cased spelling is used by GUI when creating a dataExtension based filterDefintion
+        //     isCreateable: true,
+        //     isUpdateable: false, // cannot be updated
+        //     retrieving: false,
+        //     template: false,
+        // },
         derivedFromType: {
+            // 1: SubscriberAttributes, 2: DataExtension, 6: EntryCriteria;
             isCreateable: true,
-            isUpdateable: true,
+            isUpdateable: false, // cannot be updated
             retrieving: true,
             template: true,
         },
         derivedFromObjectId: {
+            // dataExtension ID or '00000000-0000-0000-0000-000000000000' for lists
             isCreateable: true,
-            isUpdateable: true,
+            isUpdateable: false, // cannot be updated
             retrieving: true,
             template: true,
         },
         derivedFromObjectTypeName: {
-            isCreateable: true,
-            isUpdateable: true,
+            // "SubscriberAttributes" | "DataExtension" | "EntryCriteria" ...; only returned by GET API
+            isCreateable: false,
+            isUpdateable: false,
             retrieving: true,
             template: true,
         },
         derivedFromObjectName: {
+            // dataExtension name; field only returned by GET-API
             isCreateable: false,
             isUpdateable: false,
-            retrieving: true,
-            template: true,
+            retrieving: false,
+            template: false,
         },
         isSendable: {
-            isCreateable: true,
-            isUpdateable: true,
+            isCreateable: false, // automatically set during create
+            isUpdateable: false,
             retrieving: true,
             template: true,
         },

From 15548a68a45f6117477f403405177bf0d1f88a4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= <joern.berkefeld@accenture.com>
Date: Wed, 30 Aug 2023 16:02:10 +0200
Subject: [PATCH 5/8] #9: switch to folder-based collection retrieve; resolve
 all filterDefinition details; split off hidden ones, resolve filterDefinition
 in filter

---
 docs/dist/documentation.md                    | 108 +++-
 lib/MetadataTypeDefinitions.js                |   1 +
 lib/MetadataTypeInfo.js                       |   1 +
 lib/metadataTypes/DataExtension.js            |   2 +-
 lib/metadataTypes/Filter.js                   |  50 +-
 lib/metadataTypes/FilterDefinition.js         | 493 ++++++++++++++----
 lib/metadataTypes/FilterDefinitionHidden.js   |  24 +
 .../definitions/Filter.definition.js          |  19 +-
 .../FilterDefinition.definition.js            |  66 ++-
 .../FilterDefinitionHidden.definition.js      | 156 ++++++
 10 files changed, 751 insertions(+), 169 deletions(-)
 create mode 100644 lib/metadataTypes/FilterDefinitionHidden.js
 create mode 100644 lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js

diff --git a/docs/dist/documentation.md b/docs/dist/documentation.md
index 319125d8c..0df8883e8 100644
--- a/docs/dist/documentation.md
+++ b/docs/dist/documentation.md
@@ -70,6 +70,9 @@ as this is a configuration in the EID</p>
 <dt><a href="#FilterDefinition">FilterDefinition</a> ⇐ <code><a href="#MetadataType">MetadataType</a></code></dt>
 <dd><p>FilterDefinition MetadataType</p>
 </dd>
+<dt><a href="#FilterDefinitionHidden">FilterDefinitionHidden</a> ⇐ <code><a href="#FilterDefinitionHidden">FilterDefinitionHidden</a></code></dt>
+<dd><p>FilterDefinitionHidden MetadataType</p>
+</dd>
 <dt><a href="#Folder">Folder</a> ⇐ <code><a href="#MetadataType">MetadataType</a></code></dt>
 <dd><p>Folder MetadataType</p>
 </dd>
@@ -2884,8 +2887,7 @@ Filter MetadataType
 
 * [Filter](#Filter) ⇐ [<code>MetadataType</code>](#MetadataType)
     * [.retrieve(retrieveDir, [_], [__], [key])](#Filter.retrieve) ⇒ <code>Promise.&lt;{metadata: TYPE.FilterMap, type: string}&gt;</code>
-    * [.postRetrieveTasks(item)](#Filter.postRetrieveTasks) ⇒ <code>TYPE.FilterItem</code>
-    * [.parseMetadata(metadata)](#Filter.parseMetadata) ⇒ <code>TYPE.FilterItem</code>
+    * [.postRetrieveTasks(metadata)](#Filter.postRetrieveTasks) ⇒ <code>TYPE.FilterItem</code>
     * [.preDeployTasks(metadata)](#Filter.preDeployTasks) ⇒ <code>Promise.&lt;TYPE.FilterItem&gt;</code>
 
 <a name="Filter.retrieve"></a>
@@ -2908,19 +2910,7 @@ Filters with the endpoint /automation/v1/filters/{id}
 
 <a name="Filter.postRetrieveTasks"></a>
 
-### Filter.postRetrieveTasks(item) ⇒ <code>TYPE.FilterItem</code>
-manages post retrieve steps
-
-**Kind**: static method of [<code>Filter</code>](#Filter)  
-**Returns**: <code>TYPE.FilterItem</code> - parsed metadata definition  
-
-| Param | Type | Description |
-| --- | --- | --- |
-| item | <code>TYPE.FilterItem</code> | a single record |
-
-<a name="Filter.parseMetadata"></a>
-
-### Filter.parseMetadata(metadata) ⇒ <code>TYPE.FilterItem</code>
+### Filter.postRetrieveTasks(metadata) ⇒ <code>TYPE.FilterItem</code>
 parses retrieved Metadata before saving
 
 **Kind**: static method of [<code>Filter</code>](#Filter)  
@@ -2952,8 +2942,15 @@ FilterDefinition MetadataType
 
 * [FilterDefinition](#FilterDefinition) ⇐ [<code>MetadataType</code>](#MetadataType)
     * [.retrieve(retrieveDir, [_], [__], [key])](#FilterDefinition.retrieve) ⇒ <code>Promise.&lt;{metadata: TYPE.FilterDefinitionMap, type: string}&gt;</code>
+    * [.getFilterFolderIds([hidden])](#FilterDefinition.getFilterFolderIds) ⇒ <code>Array.&lt;number&gt;</code>
+    * [.getMeasureFolderIds()](#FilterDefinition.getMeasureFolderIds) ⇒ <code>Array.&lt;number&gt;</code>
+    * [.cacheDeFields(metadataTypeMapObj)](#FilterDefinition.cacheDeFields)
+    * [.cacheContactAttributes(metadataTypeMapObj)](#FilterDefinition.cacheContactAttributes)
+    * [.cacheMeasures(metadataTypeMapObj)](#FilterDefinition.cacheMeasures)
     * [.retrieveForCache()](#FilterDefinition.retrieveForCache) ⇒ <code>Promise.&lt;{metadata: TYPE.FilterDefinitionMap, type: string}&gt;</code>
     * [.postRetrieveTasks(metadata)](#FilterDefinition.postRetrieveTasks) ⇒ <code>TYPE.FilterDefinitionItem</code>
+    * [.resolveFieldIds(metadata, [fieldCache], [filter])](#FilterDefinition.resolveFieldIds) ⇒ <code>void</code>
+    * [.resolveAttributeIds(metadata, [filter])](#FilterDefinition.resolveAttributeIds) ⇒ <code>void</code>
     * [.preDeployTasks(metadata)](#FilterDefinition.preDeployTasks) ⇒ <code>Promise.&lt;TYPE.FilterDefinitionItem&gt;</code>
     * [.create(metadata)](#FilterDefinition.create) ⇒ <code>Promise.&lt;TYPE.FilterDefinitionItem&gt;</code>
     * [.update(metadata)](#FilterDefinition.update) ⇒ <code>Promise.&lt;TYPE.FilterDefinitionItem&gt;</code>
@@ -2973,6 +2970,52 @@ Retrieves all records and saves it to disk
 | [__] | <code>void</code> | unused parameter |
 | [key] | <code>string</code> | customer key of single item to retrieve |
 
+<a name="FilterDefinition.getFilterFolderIds"></a>
+
+### FilterDefinition.getFilterFolderIds([hidden]) ⇒ <code>Array.&lt;number&gt;</code>
+helper for [retrieve](#FilterDefinition.retrieve)
+
+**Kind**: static method of [<code>FilterDefinition</code>](#FilterDefinition)  
+**Returns**: <code>Array.&lt;number&gt;</code> - Array of folder IDs  
+
+| Param | Type | Default | Description |
+| --- | --- | --- | --- |
+| [hidden] | <code>boolean</code> | <code>false</code> | used to filter out hidden or non-hidden filterDefinitions |
+
+<a name="FilterDefinition.getMeasureFolderIds"></a>
+
+### FilterDefinition.getMeasureFolderIds() ⇒ <code>Array.&lt;number&gt;</code>
+helper for [retrieve](#FilterDefinition.retrieve)
+
+**Kind**: static method of [<code>FilterDefinition</code>](#FilterDefinition)  
+**Returns**: <code>Array.&lt;number&gt;</code> - Array of folder IDs  
+<a name="FilterDefinition.cacheDeFields"></a>
+
+### FilterDefinition.cacheDeFields(metadataTypeMapObj)
+**Kind**: static method of [<code>FilterDefinition</code>](#FilterDefinition)  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| metadataTypeMapObj | <code>TYPE.MultiMetadataTypeMap</code> | - |
+
+<a name="FilterDefinition.cacheContactAttributes"></a>
+
+### FilterDefinition.cacheContactAttributes(metadataTypeMapObj)
+**Kind**: static method of [<code>FilterDefinition</code>](#FilterDefinition)  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| metadataTypeMapObj | <code>TYPE.MultiMetadataTypeMap</code> | - |
+
+<a name="FilterDefinition.cacheMeasures"></a>
+
+### FilterDefinition.cacheMeasures(metadataTypeMapObj)
+**Kind**: static method of [<code>FilterDefinition</code>](#FilterDefinition)  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| metadataTypeMapObj | <code>TYPE.MultiMetadataTypeMap</code> | - |
+
 <a name="FilterDefinition.retrieveForCache"></a>
 
 ### FilterDefinition.retrieveForCache() ⇒ <code>Promise.&lt;{metadata: TYPE.FilterDefinitionMap, type: string}&gt;</code>
@@ -2992,6 +3035,27 @@ parses retrieved Metadata before saving
 | --- | --- | --- |
 | metadata | <code>TYPE.FilterDefinitionItem</code> | a single record |
 
+<a name="FilterDefinition.resolveFieldIds"></a>
+
+### FilterDefinition.resolveFieldIds(metadata, [fieldCache], [filter]) ⇒ <code>void</code>
+**Kind**: static method of [<code>FilterDefinition</code>](#FilterDefinition)  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| metadata | <code>TYPE.FilterDefinitionItem</code> | - |
+| [fieldCache] | <code>Array.&lt;object&gt;</code> | - |
+| [filter] | <code>object</code> | - |
+
+<a name="FilterDefinition.resolveAttributeIds"></a>
+
+### FilterDefinition.resolveAttributeIds(metadata, [filter]) ⇒ <code>void</code>
+**Kind**: static method of [<code>FilterDefinition</code>](#FilterDefinition)  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| metadata | <code>TYPE.FilterDefinitionItem</code> | - |
+| [filter] | <code>object</code> | - |
+
 <a name="FilterDefinition.preDeployTasks"></a>
 
 ### FilterDefinition.preDeployTasks(metadata) ⇒ <code>Promise.&lt;TYPE.FilterDefinitionItem&gt;</code>
@@ -3028,6 +3092,20 @@ Updates a single item
 | --- | --- | --- |
 | metadata | <code>TYPE.FilterDefinitionItem</code> | a single item |
 
+<a name="FilterDefinitionHidden"></a>
+
+## FilterDefinitionHidden ⇐ [<code>FilterDefinitionHidden</code>](#FilterDefinitionHidden)
+FilterDefinitionHidden MetadataType
+
+**Kind**: global class  
+**Extends**: [<code>FilterDefinitionHidden</code>](#FilterDefinitionHidden)  
+<a name="FilterDefinitionHidden.getFilterFolderIds"></a>
+
+### FilterDefinitionHidden.getFilterFolderIds() ⇒ <code>Array.&lt;number&gt;</code>
+helper for [retrieve](#FilterDefinition.retrieve)
+
+**Kind**: static method of [<code>FilterDefinitionHidden</code>](#FilterDefinitionHidden)  
+**Returns**: <code>Array.&lt;number&gt;</code> - Array of folder IDs  
 <a name="Folder"></a>
 
 ## Folder ⇐ [<code>MetadataType</code>](#MetadataType)
diff --git a/lib/MetadataTypeDefinitions.js b/lib/MetadataTypeDefinitions.js
index cba29b1db..68da505fa 100644
--- a/lib/MetadataTypeDefinitions.js
+++ b/lib/MetadataTypeDefinitions.js
@@ -23,6 +23,7 @@ const MetadataTypeDefinitions = {
     fileTransfer: require('./metadataTypes/definitions/FileTransfer.definition'),
     filter: require('./metadataTypes/definitions/Filter.definition'),
     filterDefinition: require('./metadataTypes/definitions/FilterDefinition.definition'),
+    filterDefinitionHidden: require('./metadataTypes/definitions/FilterDefinitionHidden.definition'),
     folder: require('./metadataTypes/definitions/Folder.definition'),
     importFile: require('./metadataTypes/definitions/ImportFile.definition'),
     journey: require('./metadataTypes/definitions/Journey.definition'),
diff --git a/lib/MetadataTypeInfo.js b/lib/MetadataTypeInfo.js
index f88326649..6f4112bc4 100644
--- a/lib/MetadataTypeInfo.js
+++ b/lib/MetadataTypeInfo.js
@@ -23,6 +23,7 @@ const MetadataTypeInfo = {
     fileTransfer: require('./metadataTypes/FileTransfer'),
     filter: require('./metadataTypes/Filter'),
     filterDefinition: require('./metadataTypes/FilterDefinition'),
+    filterDefinitionHidden: require('./metadataTypes/FilterDefinitionHidden'),
     folder: require('./metadataTypes/Folder'),
     importFile: require('./metadataTypes/ImportFile'),
     journey: require('./metadataTypes/Journey'),
diff --git a/lib/metadataTypes/DataExtension.js b/lib/metadataTypes/DataExtension.js
index 6e96f463b..9676723ae 100644
--- a/lib/metadataTypes/DataExtension.js
+++ b/lib/metadataTypes/DataExtension.js
@@ -1341,7 +1341,7 @@ class DataExtension extends MetadataType {
      * @returns {Promise.<{metadata: TYPE.DataExtensionMap, type: string}>} Promise
      */
     static async retrieveForCache() {
-        return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name'], this.buObject, null, null);
+        return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name']);
     }
     /**
      * Retrieves dataExtension metadata in template format.
diff --git a/lib/metadataTypes/Filter.js b/lib/metadataTypes/Filter.js
index 4a09dc1fa..214a6c3c6 100644
--- a/lib/metadataTypes/Filter.js
+++ b/lib/metadataTypes/Filter.js
@@ -33,32 +33,17 @@ class Filter extends MetadataType {
     static async retrieve(retrieveDir, _, __, key) {
         return super.retrieveREST(retrieveDir, '/automation/v1/filters/', null, key);
     }
-    /**
-     * manages post retrieve steps
-     *
-     * @param {TYPE.FilterItem} item a single record
-     * @returns {TYPE.FilterItem} parsed metadata definition
-     */
-    static postRetrieveTasks(item) {
-        return this.parseMetadata(item);
-    }
     /**
      * parses retrieved Metadata before saving
      *
      * @param {TYPE.FilterItem} metadata a single record
      * @returns {TYPE.FilterItem} parsed metadata definition
      */
-    static parseMetadata(metadata) {
-        try {
-            // folder
-            metadata.r__folder_Path = cache.searchForField(
-                'folder',
-                metadata.categoryId,
-                'ID',
-                'Path'
-            );
-            delete metadata.categoryId;
+    static postRetrieveTasks(metadata) {
+        // folder
+        this.setFolderPath(metadata);
 
+        try {
             // filterDefinition
             metadata.r__filterDefinition_CustomerKey = cache.searchForField(
                 'filterDefinition',
@@ -67,7 +52,21 @@ class Filter extends MetadataType {
                 'key'
             );
             delete metadata.filterDefinitionId;
-
+        } catch {
+            try {
+                // filterDefinition
+                metadata.r__filterDefinition_CustomerKey = cache.searchForField(
+                    'filterDefinitionHidden',
+                    metadata.filterDefinitionId,
+                    'id',
+                    'key'
+                );
+                delete metadata.filterDefinitionId;
+            } catch {
+                // ignore
+            }
+        }
+        try {
             // source
             if (metadata.sourceTypeId === 1) {
                 // list
@@ -90,7 +89,12 @@ class Filter extends MetadataType {
                     }`
                 );
             }
-
+        } catch (ex) {
+            Util.logger.warn(
+                ` - filter '${metadata.name}' (${metadata.customerKey}): Destination not found (${ex.message})`
+            );
+        }
+        try {
             // target
             if (metadata.destinationTypeId === 1) {
                 // list
@@ -106,7 +110,7 @@ class Filter extends MetadataType {
                 delete metadata.destinationTypeId;
             } else {
                 Util.logger.warn(
-                    ` - Filter '${metadata.name}' (${
+                    ` - filter '${metadata.name}' (${
                         metadata.customerKey
                     }): Unsupported destination type ${metadata.destinationTypeId}=${
                         dataTypes[metadata.destinationTypeId]
@@ -115,7 +119,7 @@ class Filter extends MetadataType {
             }
         } catch (ex) {
             Util.logger.warn(
-                ` - Filter '${metadata.name}' (${metadata.customerKey}): ${ex.message}`
+                ` - filter '${metadata.name}' (${metadata.customerKey}): Source not found (${ex.message})`
             );
         }
         return metadata;
diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js
index 5422e72eb..6c480fed2 100644
--- a/lib/metadataTypes/FilterDefinition.js
+++ b/lib/metadataTypes/FilterDefinition.js
@@ -2,6 +2,8 @@
 
 const TYPE = require('../../types/mcdev.d');
 const MetadataType = require('./MetadataType');
+const DataExtensionField = require('./DataExtensionField');
+const Folder = require('./Folder');
 const Util = require('../util/util');
 const cache = require('../util/cache');
 const { XMLBuilder, XMLParser } = require('fast-xml-parser');
@@ -12,6 +14,7 @@ const { XMLBuilder, XMLParser } = require('fast-xml-parser');
  * @augments MetadataType
  */
 class FilterDefinition extends MetadataType {
+    static cache = {}; // type internal cache for various things
     /**
      * Retrieves all records and saves it to disk
      *
@@ -22,74 +25,223 @@ class FilterDefinition extends MetadataType {
      * @returns {Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}>} Promise of items
      */
     static async retrieve(retrieveDir, _, __, key) {
-        // #1 get the list via SOAP cause the corresponding REST call has no BU filter apparently
-        // for reference the rest path: '/automation/v1/filterdefinitions?view=categoryinfo'
-
-        const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Name'];
-        let requestParams;
-        if (key) {
-            requestParams = {
-                filter: {
-                    leftOperand: 'CustomerKey',
-                    operator: 'equals',
-                    rightOperand: key,
-                },
-            };
-        }
-
-        /**
-         * @type {TYPE.FilterDefinitionSOAPItemMap[]}
-         */
-        const responseSOAP = await this.client.soap.retrieveBulk(
-            this.definition.type,
-            soapFields,
-            requestParams
-        );
+        const filterFolders = await this.getFilterFolderIds();
 
-        // backup REST value of the keyField
-        const keyFieldBak = this.definition.keyField;
-        this.definition.keyField = 'CustomerKey';
-        const responseSOAPMap = this.parseResponseBody(responseSOAP, key);
-        // restore the keyField to its REST value
-        this.definition.keyField = keyFieldBak;
-
-        /**
-         * @type {TYPE.FilterDefinitionSOAPItem[]}
-         */
-        const responseSOAPList = responseSOAP.Results.filter((item) => {
-            if (item.ObjectState) {
-                Util.logger.debug(`Filtered filterDefinition ${item.Name}: ${item.ObjectState}`);
-                return false;
-            } else {
-                return true;
-            }
-        });
-
-        // #2
-        // /automation/v1/filterdefinitions/<id>
-        const metadataMap = (
-            await super.retrieveRESTcollection(
-                responseSOAPList.map((item) => ({
-                    id: item.ObjectID,
-                    uri: '/email/v1/filters/filterdefinition/' + item.ObjectID,
-                }))
-            )
-        ).metadata;
-        for (const item of Object.values(metadataMap)) {
-            // description is not returned when empty
+        const metadataTypeMapObj = { metadata: {}, type: this.definition.type };
+        for (const folderId of filterFolders) {
+            const metadataMapFolder = await super.retrieveREST(
+                null,
+                'email/v1/filters/filterdefinition/category/' +
+                    folderId +
+                    '?derivedFromType=1,2,3,4&',
+                null,
+                key
+            );
+            if (Object.keys(metadataMapFolder.metadata).length) {
+                metadataTypeMapObj.metadata = {
+                    ...metadataTypeMapObj.metadata,
+                    ...metadataMapFolder.metadata,
+                };
+                if (key) {
+                    // if key was found we can stop checking other folders
+                    break;
+                }
+            }
+        }
+        // console.log('metadataMap', metadataMap);
+
+        for (const item of Object.values(metadataTypeMapObj.metadata)) {
+            // description is not returned when emptyg
             item.description ||= '';
-            // add extra info from XML
-            item.c__soap_DataFilter = responseSOAPMap[item.key].DataFilter;
         }
         if (retrieveDir) {
-            const savedMetadata = await this.saveResults(metadataMap, retrieveDir);
+            // custom dataExtensionField caching
+            await this.cacheDeFields(metadataTypeMapObj);
+            await this.cacheContactAttributes(metadataTypeMapObj);
+            await this.cacheMeasures(metadataTypeMapObj);
+
+            const savedMetadata = await this.saveResults(metadataTypeMapObj.metadata, retrieveDir);
             Util.logger.info(
-                `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})`
+                `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` +
+                    Util.getKeysString(key)
             );
         }
 
-        return { metadata: metadataMap, type: this.definition.type };
+        return metadataTypeMapObj;
+    }
+    /**
+     * helper for {@link FilterDefinition.retrieve}
+     *
+     * @param {boolean} [hidden] used to filter out hidden or non-hidden filterDefinitions
+     * @returns {number[]} Array of folder IDs
+     */
+    static async getFilterFolderIds(hidden = false) {
+        const fromCache =
+            this.cache.folderFilter || cache.getCache().folder
+                ? Object.values(this.cache.folderFilter || cache.getCache().folder)
+                      .filter((item) => item.ContentType === 'filterdefinition')
+                      .filter(
+                          (item) =>
+                              (!hidden && item.Path.startsWith('Data Filters')) ||
+                              (hidden && !item.Path.startsWith('Data Filters'))
+                      ) // only retrieve from Data Filters folder
+                      .map((item) => item.ID)
+                : [];
+        if (fromCache.length) {
+            return fromCache;
+        }
+
+        const subTypeArr = ['hidden', 'filterdefinition'];
+        Util.logger.info(` - Caching dependent Metadata: folder`);
+        Util.logSubtypes(subTypeArr);
+
+        Folder.client = this.client;
+        Folder.buObject = this.buObject;
+        Folder.properties = this.properties;
+        this.cache.folderFilter = (await Folder.retrieveForCache(null, subTypeArr)).metadata;
+        return this.getFilterFolderIds(hidden);
+    }
+    /**
+     * helper for {@link FilterDefinition.retrieve}
+     *
+     * @returns {number[]} Array of folder IDs
+     */
+    static async getMeasureFolderIds() {
+        const fromCache =
+            this.cache.folderMeasure?.[this.buObject.mid] || cache.getCache().folder
+                ? Object.values(
+                      this.cache.folderMeasure?.[this.buObject.mid] || cache.getCache().folder
+                  )
+                      .filter((item) => item.ContentType === 'measure')
+                      .map((item) => item.ID)
+                : [];
+        if (fromCache.length) {
+            return fromCache;
+        }
+
+        const subTypeArr = ['measure'];
+        Util.logger.info(` - Caching dependent Metadata: folder`);
+        Util.logSubtypes(subTypeArr);
+
+        Folder.client = this.client;
+        Folder.buObject = this.buObject;
+        Folder.properties = this.properties;
+        this.cache.folderMeasure ||= {};
+        this.cache.folderMeasure[this.buObject.mid] = (
+            await Folder.retrieveForCache(null, subTypeArr)
+        ).metadata;
+        return this.getMeasureFolderIds();
+    }
+
+    /**
+     *
+     * @param {TYPE.MultiMetadataTypeMap} metadataTypeMapObj -
+     */
+    static async cacheDeFields(metadataTypeMapObj) {
+        const deKeys = Object.values(metadataTypeMapObj.metadata)
+            .filter((item) => item.derivedFromObjectTypeName === 'DataExtension')
+            .filter((item) => item.derivedFromObjectId)
+            .map((item) => {
+                try {
+                    const deKey = cache.searchForField(
+                        'dataExtension',
+                        item.derivedFromObjectId,
+                        'ObjectID',
+                        'CustomerKey'
+                    );
+                    if (deKey) {
+                        this.deIdKeyMap ||= {};
+                        this.deIdKeyMap[item.derivedFromObjectId] = deKey;
+                        return deKey;
+                    }
+                } catch {
+                    return null;
+                }
+            })
+            .filter(Boolean);
+        if (deKeys.length) {
+            Util.logger.info(' - Caching dependent Metadata: dataExtensionField');
+            // only proceed with the download if we have dataExtension keys
+            const fieldOptions = {};
+            for (const deKey of deKeys) {
+                fieldOptions.filter = fieldOptions.filter
+                    ? {
+                          leftOperand: {
+                              leftOperand: 'DataExtension.CustomerKey',
+                              operator: 'equals',
+                              rightOperand: deKey,
+                          },
+                          operator: 'OR',
+                          rightOperand: fieldOptions.filter,
+                      }
+                    : {
+                          leftOperand: 'DataExtension.CustomerKey',
+                          operator: 'equals',
+                          rightOperand: deKey,
+                      };
+            }
+            DataExtensionField.buObject = this.buObject;
+            DataExtensionField.client = this.client;
+            DataExtensionField.properties = this.properties;
+            this.dataExtensionFieldCache = (
+                await DataExtensionField.retrieveForCache(fieldOptions, ['Name', 'ObjectID'])
+            ).metadata;
+        }
     }
+    /**
+     *
+     * @param {TYPE.MultiMetadataTypeMap} metadataTypeMapObj -
+     */
+    static async cacheContactAttributes(metadataTypeMapObj) {
+        if (this.cache.contactAttributes?.[this.buObject.mid]) {
+            return;
+        }
+        const subscriberFilters = Object.values(metadataTypeMapObj.metadata)
+            .filter((item) => item.derivedFromObjectTypeName === 'SubscriberAttributes')
+            .filter((item) => item.derivedFromObjectId);
+        if (subscriberFilters.length) {
+            Util.logger.info(' - Caching dependent Metadata: contactAttributes');
+            const response = await this.client.rest.get('/email/v1/Contacts/Attributes/');
+            const keyFieldBackup = this.definition.keyField;
+            this.definition.keyField = 'id';
+            this.cache.contactAttributes ||= {};
+            this.cache.contactAttributes[this.buObject.mid] = this.parseResponseBody(response);
+            this.definition.keyField = keyFieldBackup;
+        }
+    }
+    /**
+     *
+     * @param {TYPE.MultiMetadataTypeMap} metadataTypeMapObj -
+     */
+    static async cacheMeasures(metadataTypeMapObj) {
+        if (this.cache.measures?.[this.buObject.mid]) {
+            return;
+        }
+        const subscriberFilters = Object.values(metadataTypeMapObj.metadata)
+            .filter((item) => item.derivedFromObjectTypeName === 'SubscriberAttributes')
+            .filter((item) => item.derivedFromObjectId);
+        const measureFolders = await this.getMeasureFolderIds();
+        if (subscriberFilters.length) {
+            Util.logger.info(' - Caching dependent Metadata: measure');
+            const response = { items: [] };
+            for (const folderId of measureFolders) {
+                const metadataMapFolder = await this.client.rest.getBulk(
+                    'email/v1/Measures/category/' + folderId + '/',
+                    250 // 250 is what the GUI is using
+                );
+                if (Object.keys(metadataMapFolder.items).length) {
+                    response.items.push(...metadataMapFolder.items);
+                }
+            }
+
+            const keyFieldBackup = this.definition.keyField;
+            this.definition.keyField = 'measureID';
+            this.cache.measures ||= {};
+            this.cache.measures[this.buObject.mid] = this.parseResponseBody(response);
+            this.definition.keyField = keyFieldBackup;
+        }
+    }
+
     /**
      * Retrieves all records for caching
      *
@@ -111,57 +263,200 @@ class FilterDefinition extends MetadataType {
             // type 6 seems to be journey related. Maybe we need to change that again in the future
             return;
         }
-        try {
-            // folder
-            this.setFolderPath(metadata);
+        // folder
+        this.setFolderPath(metadata);
 
-            switch (metadata.derivedFromType) {
-                case 1: {
-                    // SubscriberAttributes
-                    // TODO
-                    break;
+        // parse XML filter for further processing in JSON format
+        const xmlToJson = new XMLParser({ ignoreAttributes: false });
+        metadata.c__filterDefinition = xmlToJson.parse(
+            metadata.filterDefinitionXml
+        )?.FilterDefinition;
+        delete metadata.filterDefinitionXml;
+
+        switch (metadata.derivedFromType) {
+            case 1: {
+                if (metadata.c__filterDefinition['@_Source'] === 'SubscriberAttribute') {
+                    if (
+                        metadata.derivedFromObjectId &&
+                        metadata.derivedFromObjectId !== '00000000-0000-0000-0000-000000000000'
+                    ) {
+                        // Lists
+                        try {
+                            metadata.r__source_list_PathName = cache.getListPathName(
+                                metadata.derivedFromObjectId,
+                                'ObjectID'
+                            );
+                        } catch {
+                            Util.logger.warn(
+                                ` - skipping ${this.definition.type} ${metadata.key}: list ${metadata.derivedFromObjectId} not found on current or Parent BU`
+                            );
+                            // return;
+                        }
+                    } else {
+                        // SubscriberAttributes
+                        // - nothing to do
+                    }
                 }
-                case 2: {
+
+                break;
+            }
+            case 2: {
+                // DataExtension + XXX?
+                if (
+                    metadata.c__filterDefinition['@_Source'] === 'Meta' ||
+                    metadata.derivedFromObjectId === '00000000-0000-0000-0000-000000000000'
+                ) {
+                    // TODO - weird so far not understood case of Source=Meta
+                    // sample: <FilterDefinition Source=\"Meta\"><Include><ConditionSet Operator=\"OR\" ConditionSetName=\"Individual Filter Grouping\"><Condition ID=\"55530cec-1df4-e611-80cc-1402ec7222b4\" isParam=\"false\" isPathed=\"true\"  pathAttrGroupID=\"75530cec-1df4-e611-80cc-1402ec7222b4\" Operator=\"Equal\"><Value><![CDATA[994607]]></Value></Condition><Condition ID=\"55530cec-1df4-e611-80cc-1402ec7222b4\" isParam=\"false\" isPathed=\"true\"  pathAttrGroupID=\"75530cec-1df4-e611-80cc-1402ec7222b4\" Operator=\"Equal\"><Value><![CDATA[3624804]]></Value></Condition></ConditionSet></Include><Exclude></Exclude></FilterDefinition>
+                } else if (metadata.c__filterDefinition['@_Source'] === 'DataExtension') {
                     // DataExtension
-                    metadata.r__dataExtension_CustomerKey = cache.searchForField(
-                        'dataExtension',
-                        metadata.derivedFromObjectId,
-                        'ObjectID',
-                        'CustomerKey'
-                    );
+                    try {
+                        metadata.r__source_dataExtension_CustomerKey =
+                            this.deIdKeyMap?.[metadata.derivedFromObjectId] ||
+                            cache.searchForField(
+                                'dataExtension',
+                                metadata.derivedFromObjectId,
+                                'ObjectID',
+                                'CustomerKey'
+                            );
+                    } catch {
+                        Util.logger.debug(
+                            ` - skipping ${this.definition.type} ${metadata.key}: dataExtension ${metadata.derivedFromObjectId} not found on BU`
+                        );
+                        return;
+                    }
+                }
+                break;
+            }
+            case 3: {
+                // TODO
+                break;
+            }
+            case 4: {
+                // TODO
+                break;
+            }
+            case 5: {
+                // TODO
+                break;
+            }
+            case 6: {
+                // TODO
+                break;
+            }
+        }
+
+        // map Condition ID to fields ID
+        switch (metadata.derivedFromType) {
+            case 1: {
+                // SubscriberAttributes
+                this.resolveAttributeIds(metadata);
+                delete metadata.derivedFromObjectId;
+                delete metadata.derivedFromType;
+                delete metadata.c__filterDefinition['@_Source'];
+                break;
+            }
+            case 2: {
+                if (metadata.c__filterDefinition['@_Source'] === 'Meta') {
+                    // TODO - weird so far not understood case of Source=Meta
+                    // sample: <FilterDefinition Source=\"Meta\"><Include><ConditionSet Operator=\"OR\" ConditionSetName=\"Individual Filter Grouping\"><Condition ID=\"55530cec-1df4-e611-80cc-1402ec7222b4\" isParam=\"false\" isPathed=\"true\"  pathAttrGroupID=\"75530cec-1df4-e611-80cc-1402ec7222b4\" Operator=\"Equal\"><Value><![CDATA[994607]]></Value></Condition><Condition ID=\"55530cec-1df4-e611-80cc-1402ec7222b4\" isParam=\"false\" isPathed=\"true\"  pathAttrGroupID=\"75530cec-1df4-e611-80cc-1402ec7222b4\" Operator=\"Equal\"><Value><![CDATA[3624804]]></Value></Condition></ConditionSet></Include><Exclude></Exclude></FilterDefinition>
+                } else if (metadata.c__filterDefinition['@_Source'] === 'DataExtension') {
+                    // DataExtension
+                    this.resolveFieldIds(metadata);
                     delete metadata.derivedFromObjectId;
                     delete metadata.derivedFromType;
-                    break;
-                }
-                case 3: {
-                    // TODO
-                    break;
-                }
-                case 4: {
-                    // TODO
-                    break;
-                }
-                case 5: {
-                    // TODO
-                    break;
-                }
-                case 6: {
-                    // TODO
-                    break;
+                    delete metadata.c__filterDefinition['@_Source'];
+                    delete metadata.c__filterDefinition['@_SourceID'];
                 }
+                break;
+            }
+            case 3: {
+                // TODO
+                break;
+            }
+            case 4: {
+                // TODO
+                break;
             }
+            case 5: {
+                // TODO
+                break;
+            }
+            case 6: {
+                // TODO
+                break;
+            }
+        }
+        return metadata;
+    }
 
-            const xmlToJson = new XMLParser({ ignoreAttributes: false });
-            metadata.c__filterDefinition = xmlToJson.parse(metadata.filterDefinitionXml);
-            // TODO map Condition ID to DataExtensionField ID
-            delete metadata.filterDefinitionXml;
-        } catch (ex) {
-            Util.logger.error(
-                `FilterDefinition '${metadata.name}' (${metadata.key}): ${ex.message}`
+    /**
+     *
+     * @param {TYPE.FilterDefinitionItem} metadata -
+     * @param {object[]} [fieldCache] -
+     * @param {object} [filter] -
+     * @returns {void}
+     */
+    static resolveFieldIds(metadata, fieldCache, filter) {
+        if (!filter) {
+            return this.resolveFieldIds(
+                metadata,
+                Object.values(this.dataExtensionFieldCache),
+                metadata.c__filterDefinition?.ConditionSet
             );
         }
-        return metadata;
+        const conditionsArr = Array.isArray(filter.Condition)
+            ? filter.Condition
+            : [filter.Condition];
+        for (const condition of conditionsArr) {
+            condition.r__dataExtensionField = fieldCache.find(
+                (field) => field.ObjectID === condition['@_ID']
+            )?.Name;
+            delete condition['@_ID'];
+            if (['IsEmpty', 'IsNotEmpty'].includes(condition['@_Operator'])) {
+                delete condition.Value;
+            }
+        }
+        if (filter.ConditionSet) {
+            this.resolveFieldIds(metadata, fieldCache, filter.ConditionSet);
+        }
     }
+    /**
+     *
+     * @param {TYPE.FilterDefinitionItem} metadata -
+     * @param {object} [filter] -
+     * @returns {void}
+     */
+    static resolveAttributeIds(metadata, filter) {
+        if (!filter) {
+            return this.resolveAttributeIds(metadata, metadata.c__filterDefinition?.ConditionSet);
+        }
+        const contactAttributes = this.cache.contactAttributes[this.buObject.mid];
+        const measures = this.cache.measures[this.buObject.mid];
+        const conditionsArr = Array.isArray(filter.Condition)
+            ? filter.Condition
+            : [filter.Condition];
+        for (const condition of conditionsArr) {
+            condition['@_ID'] += '';
+            if (condition['@_SourceType'] === 'Measure' && measures[condition['@_ID']]) {
+                condition.r__measure = measures[condition['@_ID']]?.name;
+                delete condition['@_ID'];
+            } else if (
+                condition['@_SourceType'] !== 'Measure' &&
+                contactAttributes[condition['@_ID']]
+            ) {
+                condition.r__contactAttribute = contactAttributes[condition['@_ID']]?.name;
+                delete condition['@_ID'];
+            }
+            if (['IsEmpty', 'IsNotEmpty'].includes(condition['@_Operator'])) {
+                delete condition.Value;
+            }
+        }
+        if (filter.ConditionSet) {
+            this.resolveAttributeIds(metadata, filter.ConditionSet);
+        }
+    }
+
     /**
      * prepares a item for deployment
      *
diff --git a/lib/metadataTypes/FilterDefinitionHidden.js b/lib/metadataTypes/FilterDefinitionHidden.js
new file mode 100644
index 000000000..634fb9a44
--- /dev/null
+++ b/lib/metadataTypes/FilterDefinitionHidden.js
@@ -0,0 +1,24 @@
+'use strict';
+
+// const TYPE = require('../../types/mcdev.d');
+const FilterDefinition = require('./FilterDefinition');
+
+/**
+ * FilterDefinitionHidden MetadataType
+ *
+ * @augments FilterDefinitionHidden
+ */
+class FilterDefinitionHidden extends FilterDefinition {
+    /**
+     * helper for {@link FilterDefinition.retrieve}
+     *
+     * @returns {number[]} Array of folder IDs
+     */
+    static async getFilterFolderIds() {
+        return super.getFilterFolderIds(true);
+    }
+}
+// Assign definition to static attributes
+FilterDefinitionHidden.definition = require('../MetadataTypeDefinitions').filterDefinitionHidden;
+
+module.exports = FilterDefinitionHidden;
diff --git a/lib/metadataTypes/definitions/Filter.definition.js b/lib/metadataTypes/definitions/Filter.definition.js
index 9f442bda1..51f50508b 100644
--- a/lib/metadataTypes/definitions/Filter.definition.js
+++ b/lib/metadataTypes/definitions/Filter.definition.js
@@ -1,8 +1,15 @@
 module.exports = {
     bodyIteratorField: 'items',
-    dependencies: ['filterDefinition', 'list', 'dataExtension', 'folder'],
+    dependencies: [
+        'filterDefinition',
+        'filterDefinitionHidden',
+        'list',
+        'dataExtension',
+        'folder-filteractivity',
+        'folder-hidden',
+    ],
     hasExtended: false,
-    idField: 'id',
+    idField: 'filterActivityId',
     keyIsFixed: null,
     keyField: 'customerKey',
     nameField: 'name',
@@ -56,10 +63,10 @@ module.exports = {
             template: true,
         },
         filterActivityId: {
-            isCreateable: null,
-            isUpdateable: null,
-            retrieving: true,
-            template: true,
+            isCreateable: false,
+            isUpdateable: true,
+            retrieving: false,
+            template: false,
         },
         filterDefinitionId: {
             isCreateable: true,
diff --git a/lib/metadataTypes/definitions/FilterDefinition.definition.js b/lib/metadataTypes/definitions/FilterDefinition.definition.js
index 83f51d78a..b8299c046 100644
--- a/lib/metadataTypes/definitions/FilterDefinition.definition.js
+++ b/lib/metadataTypes/definitions/FilterDefinition.definition.js
@@ -1,5 +1,5 @@
 module.exports = {
-    bodyIteratorField: 'Results',
+    bodyIteratorField: 'items',
     dependencies: ['folder-filterdefinition', 'folder-hidden', 'dataExtension'],
     filter: {},
     hasExtended: false,
@@ -12,10 +12,10 @@ module.exports = {
     createdNameField: 'createdBy',
     lastmodDateField: 'lastUpdated',
     lastmodNameField: 'lastUpdatedBy',
-    restPagination: false,
+    restPagination: true,
+    restPageSize: 100,
     type: 'filterDefinition',
-    typeDescription:
-        'Defines an audience based on specified rules. Used by Filter Activities and Filtered DEs.',
+    typeDescription: 'Defines an audience based on specified rules. Used by Filter Activities.',
     typeRetrieveByDefault: true,
     typeName: 'Filter Definition',
     fields: {
@@ -44,12 +44,13 @@ module.exports = {
             retrieving: false,
             template: false,
         },
-        // createdByName: {
-        //     isCreateable: false,
-        //     isUpdateable: false,
-        //     retrieving: false,
-        //     template: false,
-        // },
+        createdByName: {
+            // actual name of user indicated by id in createdBy
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
         lastUpdated: {
             isCreateable: false,
             isUpdateable: false,
@@ -62,12 +63,13 @@ module.exports = {
             retrieving: false,
             template: false,
         },
-        // lastUpdatedName: {
-        //     isCreateable: false,
-        //     isUpdateable: false,
-        //     retrieving: false,
-        //     template: false,
-        // },
+        lastUpdatedByName: {
+            // actual name of user indicated by id in lastUpdatedBy
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
         name: {
             isCreateable: true,
             isUpdateable: true,
@@ -81,13 +83,12 @@ module.exports = {
             retrieving: true,
             template: true,
         },
-        // CategoryId: {
-        //     // used by UPDATE payload
-        //     isCreateable: false,
-        //     isUpdateable: true,
-        //     retrieving: false,
-        //     template: false,
-        // },
+        description: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
         filterDefinitionXml: {
             isCreateable: true,
             isUpdateable: true,
@@ -126,8 +127,8 @@ module.exports = {
             // dataExtension name; field only returned by GET-API
             isCreateable: false,
             isUpdateable: false,
-            retrieving: false,
-            template: false,
+            retrieving: true,
+            template: true,
         },
         isSendable: {
             isCreateable: false, // automatically set during create
@@ -135,5 +136,20 @@ module.exports = {
             retrieving: true,
             template: true,
         },
+        r__dataExtension_CustomerKey: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true,
+            template: true,
+        },
+        c__filterDefinition: {
+            skipValidation: true,
+        },
+        r__folder_Path: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true,
+            template: true,
+        },
     },
 };
diff --git a/lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js b/lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js
new file mode 100644
index 000000000..60b8a7c70
--- /dev/null
+++ b/lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js
@@ -0,0 +1,156 @@
+module.exports = {
+    bodyIteratorField: 'items',
+    dependencies: ['folder-filterdefinition', 'folder-hidden', 'dataExtension', 'list'],
+    filter: {},
+    hasExtended: false,
+    idField: 'id',
+    keyField: 'key',
+    nameField: 'name',
+    folderType: 'filterdefinition',
+    folderIdField: 'categoryId',
+    createdDateField: 'createdDate',
+    createdNameField: 'createdBy',
+    lastmodDateField: 'lastUpdated',
+    lastmodNameField: 'lastUpdatedBy',
+    restPagination: true,
+    restPageSize: 100,
+    type: 'filterDefinitionHidden',
+    typeDescription:
+        'Defines an audience based on specified rules. Used by filtered DEs and filtered Lists.',
+    typeRetrieveByDefault: false,
+    typeName: 'Filter Definition for filtered Lists && Data Extensions',
+    fields: {
+        // the GUI seems to ONLY send fields during update that are actually changed. It has yet to be tested if that also works with sending other fiedls as well
+        id: {
+            isCreateable: false,
+            isUpdateable: false, // included in URL
+            retrieving: false,
+            template: false,
+        },
+        key: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        createdDate: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        createdBy: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        createdByName: {
+            // actual name of user indicated by id in createdBy
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        lastUpdated: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        lastUpdatedBy: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        lastUpdatedByName: {
+            // actual name of user indicated by id in lastUpdatedBy
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: false,
+            template: false,
+        },
+        name: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        categoryId: {
+            // returned by GET / CREATE / UPDATE; used in CREATE payload
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        description: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        filterDefinitionXml: {
+            isCreateable: true,
+            isUpdateable: true,
+            retrieving: true,
+            template: true,
+        },
+        // DerivedFromType: {
+        //     // this upper-cased spelling is used by GUI when creating a dataExtension based filterDefintion
+        //     isCreateable: true,
+        //     isUpdateable: false, // cannot be updated
+        //     retrieving: false,
+        //     template: false,
+        // },
+        derivedFromType: {
+            // 1: SubscriberAttributes, 2: DataExtension, 6: EntryCriteria;
+            isCreateable: true,
+            isUpdateable: false, // cannot be updated
+            retrieving: true,
+            template: true,
+        },
+        derivedFromObjectId: {
+            // dataExtension ID or '00000000-0000-0000-0000-000000000000' for lists
+            isCreateable: true,
+            isUpdateable: false, // cannot be updated
+            retrieving: true,
+            template: true,
+        },
+        derivedFromObjectTypeName: {
+            // "SubscriberAttributes" | "DataExtension" | "EntryCriteria" ...; only returned by GET API
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true,
+            template: true,
+        },
+        derivedFromObjectName: {
+            // dataExtension name; field only returned by GET-API
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true,
+            template: true,
+        },
+        isSendable: {
+            isCreateable: false, // automatically set during create
+            isUpdateable: false,
+            retrieving: true,
+            template: true,
+        },
+        r__dataExtension_CustomerKey: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true,
+            template: true,
+        },
+        c__filterDefinition: {
+            skipValidation: true,
+        },
+        r__folder_Path: {
+            isCreateable: false,
+            isUpdateable: false,
+            retrieving: true,
+            template: true,
+        },
+    },
+};

From 8cb78b059b648a84c8fec74dcb2f02db21754b6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= <joern.berkefeld@accenture.com>
Date: Wed, 30 Aug 2023 17:13:21 +0200
Subject: [PATCH 6/8] #9: make filters available to type automation

---
 lib/metadataTypes/definitions/Automation.definition.js | 1 +
 lib/metadataTypes/definitions/Filter.definition.js     | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/lib/metadataTypes/definitions/Automation.definition.js b/lib/metadataTypes/definitions/Automation.definition.js
index 656bcf659..d6054bb9f 100644
--- a/lib/metadataTypes/definitions/Automation.definition.js
+++ b/lib/metadataTypes/definitions/Automation.definition.js
@@ -27,6 +27,7 @@ module.exports = {
         'dataExtract',
         'emailSend',
         'fileTransfer',
+        'filter',
         'folder-automations',
         'importFile',
         'query',
diff --git a/lib/metadataTypes/definitions/Filter.definition.js b/lib/metadataTypes/definitions/Filter.definition.js
index 51f50508b..5fbea7be4 100644
--- a/lib/metadataTypes/definitions/Filter.definition.js
+++ b/lib/metadataTypes/definitions/Filter.definition.js
@@ -13,6 +13,8 @@ module.exports = {
     keyIsFixed: null,
     keyField: 'customerKey',
     nameField: 'name',
+    folderType: 'filteractivity',
+    folderIdField: 'categoryId',
     createdDateField: 'createdDate',
     createdNameField: null,
     lastmodDateField: 'modifiedDate',

From 45f8657ba77f75fa8f5f9f7d74b7dad9dc9d02cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= <joern.berkefeld@accenture.com>
Date: Wed, 30 Aug 2023 17:17:18 +0200
Subject: [PATCH 7/8] #325: save notificationEmailAddress as array of emails to
 enhance UX

---
 lib/metadataTypes/Verification.js | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/metadataTypes/Verification.js b/lib/metadataTypes/Verification.js
index 93ce29fd0..6e967521b 100644
--- a/lib/metadataTypes/Verification.js
+++ b/lib/metadataTypes/Verification.js
@@ -189,6 +189,10 @@ class Verification extends MetadataType {
             'ObjectID'
         );
         delete metadata.r__dataExtension_CustomerKey;
+        metadata.notificationEmailAddress = Array.isArray(metadata.notificationEmailAddress)
+            ? metadata.notificationEmailAddress.map((item) => item.trim()).join(',')
+            : Array.isArray(metadata.notificationEmailAddress);
+
         return metadata;
     }
     /**
@@ -211,6 +215,9 @@ class Verification extends MetadataType {
                 ` - ${this.definition.type} ${metadata[this.definition.keyField]}: ${ex.message}`
             );
         }
+        metadata.notificationEmailAddress = metadata.notificationEmailAddress
+            .split(',')
+            .map((item) => item.trim());
         return metadata;
     }
     /**

From e0a4655cf2d91e42812c6bcd792b10e397eb6c8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= <joern.berkefeld@accenture.com>
Date: Wed, 30 Aug 2023 17:23:41 +0200
Subject: [PATCH 8/8] #325: adapt tests to change in
 45f8657ba77f75fa8f5f9f7d74b7dad9dc9d02cb

---
 test/resources/9999999/verification/build-expected.json    | 2 +-
 test/resources/9999999/verification/get-expected.json      | 2 +-
 test/resources/9999999/verification/patch-expected.json    | 2 +-
 test/resources/9999999/verification/post-expected.json     | 2 +-
 test/resources/9999999/verification/template-expected.json | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/resources/9999999/verification/build-expected.json b/test/resources/9999999/verification/build-expected.json
index f717cc626..98d32e01c 100644
--- a/test/resources/9999999/verification/build-expected.json
+++ b/test/resources/9999999/verification/build-expected.json
@@ -1,6 +1,6 @@
 {
     "dataVerificationDefinitionId": "testTemplated_39f6a488-20eb-4ba0-b0b9",
-    "notificationEmailAddress": "",
+    "notificationEmailAddress": [""],
     "notificationEmailMessage": "",
     "r__dataExtension_CustomerKey": "testTemplated_dataExtension",
     "shouldEmailOnFailure": false,
diff --git a/test/resources/9999999/verification/get-expected.json b/test/resources/9999999/verification/get-expected.json
index 1f9c1825b..1be889a98 100644
--- a/test/resources/9999999/verification/get-expected.json
+++ b/test/resources/9999999/verification/get-expected.json
@@ -1,6 +1,6 @@
 {
     "dataVerificationDefinitionId": "testExisting_39f6a488-20eb-4ba0-b0b9",
-    "notificationEmailAddress": "",
+    "notificationEmailAddress": [""],
     "notificationEmailMessage": "",
     "r__dataExtension_CustomerKey": "testExisting_dataExtension",
     "shouldEmailOnFailure": false,
diff --git a/test/resources/9999999/verification/patch-expected.json b/test/resources/9999999/verification/patch-expected.json
index dc8d975d2..dde339242 100644
--- a/test/resources/9999999/verification/patch-expected.json
+++ b/test/resources/9999999/verification/patch-expected.json
@@ -1,6 +1,6 @@
 {
     "dataVerificationDefinitionId": "testExisting_39f6a488-20eb-4ba0-b0b9",
-    "notificationEmailAddress": "test@accenture.com",
+    "notificationEmailAddress": ["test@accenture.com"],
     "notificationEmailMessage": "",
     "r__dataExtension_CustomerKey": "testExisting_dataExtension",
     "shouldEmailOnFailure": true,
diff --git a/test/resources/9999999/verification/post-expected.json b/test/resources/9999999/verification/post-expected.json
index b06a77b76..41795da1f 100644
--- a/test/resources/9999999/verification/post-expected.json
+++ b/test/resources/9999999/verification/post-expected.json
@@ -6,6 +6,6 @@
     "value2": 0,
     "shouldStopOnFailure": false,
     "shouldEmailOnFailure": false,
-    "notificationEmailAddress": "",
+    "notificationEmailAddress": [""],
     "notificationEmailMessage": ""
 }
diff --git a/test/resources/9999999/verification/template-expected.json b/test/resources/9999999/verification/template-expected.json
index 04296b7ea..f75056e69 100644
--- a/test/resources/9999999/verification/template-expected.json
+++ b/test/resources/9999999/verification/template-expected.json
@@ -1,6 +1,6 @@
 {
     "dataVerificationDefinitionId": "{{{prefix}}}39f6a488-20eb-4ba0-b0b9",
-    "notificationEmailAddress": "",
+    "notificationEmailAddress": [""],
     "notificationEmailMessage": "",
     "r__dataExtension_CustomerKey": "{{{prefix}}}dataExtension",
     "shouldEmailOnFailure": false,