From 8cb74e148b9fac70bb1806c380b7bbe48caf2cba Mon Sep 17 00:00:00 2001 From: prijendev Date: Thu, 25 Aug 2022 15:07:16 +0530 Subject: [PATCH 1/8] Added `ticket_fields` stream. --- tap_freshdesk/schemas/ticket_fields.json | 79 ++++++++++++++++++++++++ tap_freshdesk/streams.py | 9 ++- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 tap_freshdesk/schemas/ticket_fields.json diff --git a/tap_freshdesk/schemas/ticket_fields.json b/tap_freshdesk/schemas/ticket_fields.json new file mode 100644 index 0000000..a67b901 --- /dev/null +++ b/tap_freshdesk/schemas/ticket_fields.json @@ -0,0 +1,79 @@ +{ + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "description": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "required_for_closure": { + "type": ["null", "boolean"] + }, + "type": { + "type": ["null", "string"] + }, + "required_for_agents": { + "type": ["null", "boolean"] + }, + "required_for_customers": { + "type": ["null", "boolean"] + }, + "label_for_customers": { + "type": ["null", "string"] + }, + "customers_can_edit": { + "type": ["null", "boolean"] + }, + "displayed_to_customers": { + "type": ["null", "boolean"] + }, + "portal_cc": { + "type": ["null", "boolean"] + }, + "portal_cc_to": { + "type": ["null", "string"] + }, + "choices": {}, + "is_fsm": { + "type": ["null", "boolean"] + }, + "field_update_in_progress": { + "type": ["null", "boolean"] + }, + "dependent_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": {} + } + }, + "section_mappings": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": {} + } + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/tap_freshdesk/streams.py b/tap_freshdesk/streams.py index 8e2d715..4da8bbc 100644 --- a/tap_freshdesk/streams.py +++ b/tap_freshdesk/streams.py @@ -245,6 +245,12 @@ class TimeEntries(ChildStream): path = 'tickets/{}/time_entries' parent = 'tickets' +class TicketFields(Stream): + tap_stream_id = 'ticket_fields' + key_properties = ['id'] + replication_keys = ['updated_at'] + replication_method = 'INCREMENTAL' + path = 'ticket_fields' STREAMS = { "agents": Agents, @@ -254,5 +260,6 @@ class TimeEntries(ChildStream): "roles": Roles, "satisfaction_ratings": SatisfactionRatings, "tickets": Tickets, - "time_entries": TimeEntries + "time_entries": TimeEntries, + 'ticket_fields': TicketFields } From efa6da6402798234165fb3d1c9d630785fbccf2d Mon Sep 17 00:00:00 2001 From: prijendev Date: Fri, 26 Aug 2022 11:21:49 +0530 Subject: [PATCH 2/8] Updated tap-tester test cases. --- tests/base.py | 6 ++++++ tests/test_freshdesk_pagination.py | 4 ++-- tests/test_freshdesk_start_date.py | 20 ++++++++++++++++---- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/base.py b/tests/base.py index ad10329..b83a6b0 100644 --- a/tests/base.py +++ b/tests/base.py @@ -125,6 +125,12 @@ def expected_metadata(self): self.REPLICATION_KEYS: {"updated_at"}, self.OBEYS_START_DATE: True }, + "ticket_fields": { + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {"updated_at"}, + self.OBEYS_START_DATE: True + } } ############################# diff --git a/tests/test_freshdesk_pagination.py b/tests/test_freshdesk_pagination.py index 12ad143..2345808 100644 --- a/tests/test_freshdesk_pagination.py +++ b/tests/test_freshdesk_pagination.py @@ -20,7 +20,7 @@ def test_run(self): # Instantiate connection conn_id = connections.ensure_connection(self) - # To collect "time_entries", "satisfaction_ratings" pro account is needed. Skipping them for now. + # Add supported streams 1 by 1 expected_streams = self.expected_streams() - {"time_entries", "satisfaction_ratings"} found_catalogs = self.run_and_verify_check_mode(conn_id) @@ -43,7 +43,7 @@ def test_run(self): with self.subTest(stream=stream): # Not able to generate more data as roles stream requires pro account. # So, updating page_sie according to data available. - if stream == "roles": + if stream == "roles" or stream == "ticket_fields": page_size = 2 else: page_size = 100 diff --git a/tests/test_freshdesk_start_date.py b/tests/test_freshdesk_start_date.py index 6927519..201e52f 100644 --- a/tests/test_freshdesk_start_date.py +++ b/tests/test_freshdesk_start_date.py @@ -17,6 +17,21 @@ def name(): return "tap_tester_freshdesk_start_date_test" def test_run(self): + # Streams to verify start date tests + expected_streams = self.expected_streams() + + # To collect "time_entries", "satisfaction_ratings" pro account is needed. Skipping them for now. + expected_streams = expected_streams - {'satisfaction_ratings', 'time_entries'} + + # running start_date_test for `ticket_fields` stream + expected_stream_1 = {"ticket_fields"} + self.run_start_date(expected_stream_1, "2019-07-19T00:00:00Z") + + # running start_date_test for rest of the streams + expected_streams = expected_streams - expected_stream_1 + self.run_start_date(expected_streams, "2022-07-19T00:00:00Z") + + def run_start_date(self, expected_streams, new_start_date): """ • Verify that a sync with a later start date has at least one record synced and less records than the 1st sync with a previous start date @@ -28,16 +43,13 @@ def test_run(self): """ self.start_date_1 = self.get_properties().get('start_date') - self.start_date_2 = "2022-07-19T00:00:00Z" + self.start_date_2 = new_start_date self.start_date = self.start_date_1 start_date_1_epoch = self.dt_to_ts(self.start_date_1, self.START_DATE_FORMAT) start_date_2_epoch = self.dt_to_ts(self.start_date_2, self.START_DATE_FORMAT) - # To collect "time_entries", "satisfaction_ratings" pro account is needed. Skipping them for now. - expected_streams = self.expected_streams() - {'satisfaction_ratings', 'time_entries'} - ########################################################################## ### First Sync ########################################################################## From a371559c7c17914fabc3856fc1f1e19de3ccfeda Mon Sep 17 00:00:00 2001 From: prijendev Date: Fri, 26 Aug 2022 11:32:29 +0530 Subject: [PATCH 3/8] Updated pagination test. --- tests/test_freshdesk_pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_freshdesk_pagination.py b/tests/test_freshdesk_pagination.py index 2345808..4d266c0 100644 --- a/tests/test_freshdesk_pagination.py +++ b/tests/test_freshdesk_pagination.py @@ -20,7 +20,7 @@ def test_run(self): # Instantiate connection conn_id = connections.ensure_connection(self) - # Add supported streams 1 by 1 + # To collect "time_entries", "satisfaction_ratings" pro account is needed. Skipping them for now. expected_streams = self.expected_streams() - {"time_entries", "satisfaction_ratings"} found_catalogs = self.run_and_verify_check_mode(conn_id) From 4c47ce58e9dc7b1098a6f200aa41225ce7f3774f Mon Sep 17 00:00:00 2001 From: prijendev Date: Fri, 26 Aug 2022 11:58:39 +0530 Subject: [PATCH 4/8] Updated pagination test case. --- tests/test_freshdesk_pagination.py | 2 +- tests/test_freshdesk_start_date.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_freshdesk_pagination.py b/tests/test_freshdesk_pagination.py index 4d266c0..ac871d2 100644 --- a/tests/test_freshdesk_pagination.py +++ b/tests/test_freshdesk_pagination.py @@ -43,7 +43,7 @@ def test_run(self): with self.subTest(stream=stream): # Not able to generate more data as roles stream requires pro account. # So, updating page_sie according to data available. - if stream == "roles" or stream == "ticket_fields": + if stream in ["roles", "ticket_fields"]: page_size = 2 else: page_size = 100 diff --git a/tests/test_freshdesk_start_date.py b/tests/test_freshdesk_start_date.py index 201e52f..4125b0e 100644 --- a/tests/test_freshdesk_start_date.py +++ b/tests/test_freshdesk_start_date.py @@ -23,11 +23,11 @@ def test_run(self): # To collect "time_entries", "satisfaction_ratings" pro account is needed. Skipping them for now. expected_streams = expected_streams - {'satisfaction_ratings', 'time_entries'} - # running start_date_test for `ticket_fields` stream + # Running start_date_test for `ticket_fields` stream expected_stream_1 = {"ticket_fields"} self.run_start_date(expected_stream_1, "2019-07-19T00:00:00Z") - # running start_date_test for rest of the streams + # Running start_date_test for rest of the streams expected_streams = expected_streams - expected_stream_1 self.run_start_date(expected_streams, "2022-07-19T00:00:00Z") From 8077833e7cbbb860751806c79f1c51a39dcfd61a Mon Sep 17 00:00:00 2001 From: prijendev Date: Wed, 21 Sep 2022 11:50:19 +0530 Subject: [PATCH 5/8] Updated ReadME.md --- README.md | 1 + tap_freshdesk/streams.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 1f5849a..af6abeb 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ This tap: - [Satisfaction Ratings](https://developer.freshdesk.com/api/#satisfaction-ratings) - [Tickets](https://developer.freshdesk.com/api/#tickets) - [Time Entries](https://developer.freshdesk.com/api/#time-entries) + - [Ticket Fields](https://developer.freshdesk.com/api/#ticket-fields) - Outputs the schema for each resource - Incrementally pulls data based on the input state diff --git a/tap_freshdesk/streams.py b/tap_freshdesk/streams.py index 642b935..e58f15d 100644 --- a/tap_freshdesk/streams.py +++ b/tap_freshdesk/streams.py @@ -331,6 +331,9 @@ class TimeEntries(ChildStream): parent = 'tickets' class TicketFields(Stream): + """ + https://developer.freshdesk.com/api/#list_all_ticket_fields + """ tap_stream_id = 'ticket_fields' key_properties = ['id'] replication_keys = ['updated_at'] From d484fc9a51aadeb30a86be4f47f5e3ee0f2c5084 Mon Sep 17 00:00:00 2001 From: prijendev Date: Wed, 21 Sep 2022 12:24:33 +0530 Subject: [PATCH 6/8] Updated pagination test for ticket_fields stream. --- tests/test_freshdesk_pagination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_freshdesk_pagination.py b/tests/test_freshdesk_pagination.py index 9af1ee8..15aa3d7 100644 --- a/tests/test_freshdesk_pagination.py +++ b/tests/test_freshdesk_pagination.py @@ -26,7 +26,7 @@ def test_run(self): """ # For roles stream data present in test account is limited. So, adding configurable page_size "2" - streams_to_test_1 = {"roles"} + streams_to_test_1 = {"roles", "ticket_fields"} self.run_test(streams_to_test_1, 2) streams_to_test_2 = self.expected_streams(only_trial_account_streams=True) - streams_to_test_1 From c670ff1ae70776c8ef8a1e2ef35b07c86c7fce2c Mon Sep 17 00:00:00 2001 From: prijendev Date: Wed, 21 Sep 2022 18:41:56 +0530 Subject: [PATCH 7/8] Updated package_data to include all schemas in setup.py --- setup.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index cbbc079..cf49714 100644 --- a/setup.py +++ b/setup.py @@ -25,16 +25,8 @@ ''', packages=['tap_freshdesk'], package_data = { - 'tap_freshdesk/schemas': [ - 'agents.json', - 'companies.json', - 'contacts.json', - 'conversations.json', - 'groups.json', - 'roles.json', - 'satisfaction_ratings.json', - 'tickets.json', - 'time_entries.json', + 'tap_freshdesk': [ + 'schemas/*.json' ], }, include_package_data=True, From 5ba08894a97139f538d4598e012359ac739db0c7 Mon Sep 17 00:00:00 2001 From: prijendev Date: Tue, 27 Sep 2022 11:35:25 +0530 Subject: [PATCH 8/8] Enhanced tap tester test cases. --- tests/test_freshdesk_start_date.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_freshdesk_start_date.py b/tests/test_freshdesk_start_date.py index b59d5e0..1f06c6b 100644 --- a/tests/test_freshdesk_start_date.py +++ b/tests/test_freshdesk_start_date.py @@ -14,11 +14,11 @@ def name(): return "tap_tester_freshdesk_start_date_test" def test_run(self): - # Streams to verify start date tests - expected_streams = self.expected_streams() - + """ + Run start date test with corresponding start_date according to data availability for each stream. + """ # To collect "time_entries", "satisfaction_ratings" pro account is needed. Skipping them for now. - expected_streams = expected_streams - {'satisfaction_ratings', 'time_entries'} + expected_streams = self.expected_streams() - {'satisfaction_ratings', 'time_entries'} # Running start_date_test for `ticket_fields` stream expected_stream_1 = {"ticket_fields"}