Merge pull request #4 from MarletteFunding/feature/data-2830

Feature/data 2830

.circleci/config.yml

@@ -21,7 +21,7 @@ jobs:
           command: |
             source dev_env.sh
             source /usr/local/share/virtualenvs/tap-zendesk-chat/bin/activate
-            pylint tap_zendesk_chat -d "$PYLINT_DISABLE_LIST,no-self-use"
+            pylint tap_zendesk_chat -d "$PYLINT_DISABLE_LIST"
       - run:
           name: 'JSON Validator'
           command: |

.github/pull_request_template.md

@@ -9,3 +9,7 @@
 
 # Rollback steps
  - revert this branch
+
+#### AI generated code
+https://internal.qlik.dev/general/ways-of-working/code-reviews/#guidelines-for-ai-generated-code
+- [ ] this PR has been written with the help of GitHub Copilot or another generative AI tool

CHANGELOG.md

@@ -1,6 +1,16 @@
 # Changelog
-## 0.4.0
 
+## 0.5.1
+  * Bug Fix with chats stream [#54](https://github.com/singer-io/tap-zendesk-chat/pull/54)
+  * Dependabot update
+
+## 0.5.0
+  * Zendesk Domain Change [#52](https://github.com/singer-io/tap-zendesk-chat/pull/52)
+
+## 0.4.1
+  * Dependabot update [#50](https://github.com/singer-io/tap-zendesk-chat/pull/50)
+
+## 0.4.0
   * Code Refactoring [#45](https://github.com/singer-io/tap-zendesk-chat/pull/45)
     - Improved directory structure
     - Added pagination support to BANS stream
@@ -52,4 +62,4 @@
   * Lowers the chat interval days retrieved to 14 to account for a 10k search result limit [#8](https://github.com/singer-io/tap-zendesk-chat/pull/8)
 
 ## 0.1.12
-  * Allow chat interval days to be specified as a string in the config [#10](https://github.com/singer-io/tap-zendesk-chat/pull/10)
+  * Allow chat interval days to be specified as a string in the config [#10](https://github.com/singer-io/tap-zendesk-chat/pull/10)

setup.py

@@ -3,15 +3,15 @@
 
 setup(
     name="tap-zendesk-chat",
-    version="0.4.0",
+    version="0.5.1",
     description="Singer.io tap for extracting data from the Zendesk Chat API",
     author="Stitch",
     url="https://singer.io",
     classifiers=["Programming Language :: Python :: 3 :: Only"],
     py_modules=["tap_zendesk_chat"],
     install_requires=[
         "singer-python==5.12.1",
-        "requests==2.20.0",
+        "requests==2.32.3",
     ],
     extras_require={"dev": ["pylint", "ipdb", "nose"]},
     entry_points="""

tap_zendesk_chat/__init__.py

@@ -23,3 +23,4 @@ def main():
 
 if __name__ == "__main__":
     main()
+

tap_zendesk_chat/context.py

@@ -49,3 +49,4 @@ def update_start_date_bookmark(self, path):
 
     def write_state(self):
         write_state(self.state)
+

tap_zendesk_chat/discover.py

@@ -65,3 +65,4 @@ def discover(config: dict) -> Catalog:
             }
         )
     return Catalog.from_dict({"streams": streams})
+

tap_zendesk_chat/http_client.py

@@ -9,21 +9,54 @@
 class RateLimitException(Exception):
     pass
 
+class InvalidConfigurationError(Exception):
+    pass
+
 
 class Client:
     def __init__(self, config):
         self.access_token = config["access_token"]
         self.user_agent = config.get("user_agent", "tap-zendesk-chat")
         self.headers = {}
+        self.subdomain = config.get("subdomain")
         self.headers["Authorization"] = f"Bearer {self.access_token}"
         self.headers["User-Agent"] = self.user_agent
+        self.base_url = self.get_base_url()
         self.session = requests.Session()
 
+    def get_base_url(self):
+        """
+        Determines the base URL to use for Zendesk API requests.
+
+        Checks the availability of zendesk chat endpoints
+        and returns the available one
+        Returns:
+            str: The base URL to use for subsequent API requests.
+
+        Raises:
+            InvalidConfigurationError: If neither endpoint is accessible.
+        """
+        urls = [
+            (f"https://{self.subdomain}.zendesk.com" , "/api/v2/chat/agents"),
+            (BASE_URL , "/api/v2/agents")
+        ]
+        if not self.subdomain:
+            # return base url incase of missing subdomain
+            return BASE_URL
+        for domain, endpoint in urls:
+            resp = requests.get(f"{domain}{endpoint}", headers=self.headers, timeout=25)
+            LOGGER.info("API CHECK %s %s", resp.url, resp.status_code)
+            if resp.status_code == 200:
+                return domain
+        raise InvalidConfigurationError("Please check the URL or reauthenticate")
+
     @backoff.on_exception(backoff.expo, RateLimitException, max_tries=10, factor=2)
     def request(self, tap_stream_id, params=None, url=None, url_extra=""):
         with metrics.http_request_timer(tap_stream_id) as timer:
-
-            url = url or f"{BASE_URL}/api/v2/{tap_stream_id}{url_extra}"
+            if self.base_url == BASE_URL:
+                url = url or f"{self.base_url}/api/v2/{tap_stream_id}{url_extra}"
+            else:
+                url = url or f"{self.base_url}/api/v2/chat/{tap_stream_id}{url_extra}"
             LOGGER.info("calling %s %s", url, params)
             response = self.session.get(url, headers=self.headers, params=params)
             timer.tags[metrics.Tag.http_status_code] = response.status_code
@@ -37,3 +70,4 @@ def request(self, tap_stream_id, params=None, url=None, url_extra=""):
             )
         response.raise_for_status()
         return response.json()
+

tap_zendesk_chat/schemas/chat_engagement.json

@@ -102,3 +102,4 @@
     "object"
   ]
 }
+

tap_zendesk_chat/schemas/chat_history.json

@@ -163,3 +163,4 @@
     "chat_conversion"
   ]
 }
+

tap_zendesk_chat/schemas/chats.json

@@ -258,3 +258,4 @@
     "chat_engagement"
   ]
 }
+

tap_zendesk_chat/streams.py

@@ -116,7 +116,7 @@ def _bulk_chats(self, ctx, chat_ids: List):
         body = ctx.client.request(self.tap_stream_id, params=params)
         return list(body["docs"].values())
 
-    def _pull(self, ctx, chat_type, ts_field, full_sync, schema: Dict, stream_metadata: Dict, transformer: Transformer):
+    def _pull(self, *, ctx, chat_type, ts_field, full_sync, schema: Dict, stream_metadata: Dict, transformer: Transformer):
         """Pulls and writes pages of data for the given chat_type, where
         chat_type can be either "chat" or "offline_msg".
 
@@ -181,18 +181,18 @@ def _should_run_full_sync(self, ctx) -> bool:
     def sync(self, ctx, schema: Dict, stream_metadata: Dict, transformer: Transformer):
         full_sync = self._should_run_full_sync(ctx)
         self._pull(
-            ctx,
-            "chat",
-            "end_timestamp",
+            ctx=ctx,
+            chat_type="chat",
+            ts_field="end_timestamp",
             full_sync=full_sync,
             schema=schema,
             stream_metadata=stream_metadata,
             transformer=transformer,
         )
         self._pull(
-            ctx,
-            "offline_msg",
-            "timestamp",
+            ctx=ctx,
+            chat_type="offline_msg",
+            ts_field="timestamp",
             full_sync=full_sync,
             schema=schema,
             stream_metadata=stream_metadata,
@@ -237,3 +237,4 @@ class Triggers(BaseStream):
     Shortcuts.tap_stream_id: Shortcuts,
     Triggers.tap_stream_id: Triggers,
 }
+

tap_zendesk_chat/sync.py

@@ -29,3 +29,4 @@ def sync(ctx):
 
     ctx.state = set_currently_syncing(ctx.state, None)
     write_state(ctx.state)
+

tests/unittests/test_auth_discovery.py

@@ -13,6 +13,7 @@ def __init__(self):
         self.discover = True
         self.properties = False
         self.config = {"access_token": "abc-def"}
+        self.config_wd_subdomain = {"access_token": "abc-def", "subdomain":"test"}
         self.state = False
         self.properties = {}
 
@@ -43,9 +44,16 @@ def test_basic_auth_no_access_401(self):
 
         with self.assertRaises(HTTPError) as e:
             tap_zendesk_chat.discover(args.config)
-        # Verifying the message formed for the custom exception
-        expected_error_message = "401 Client Error: Unauthorized for url:"
-        self.assertIn(expected_error_message, str(e.exception))
+            # Verifying the message formed for the custom exception
+            expected_error_message = "401 Client Error: Unauthorized for url:"
+            self.assertIn(expected_error_message, str(e.exception))
+
+        expected_error_message = "Please check the URL or reauthenticate"
+
+        with self.assertRaises(InvalidConfigurationError) as e:
+            tap_zendesk_chat.discover(args.config_wd_subdomain)
+            self.assertIn(expected_error_message, str(e.exception))
+
 
     @mock.patch("tap_zendesk_chat.utils", return_value=Args())
     @mock.patch("singer.catalog.Catalog.from_dict", return_value={"key": "value"})
@@ -63,6 +71,7 @@ def test_discovery(self, mock_utils, mock_catalog, mock_request):
         self.assertEqual(tap_zendesk_chat.discover(Args().config), expected)
 
 
+
 class TestAccountEndpointAuthorized(unittest.TestCase):
     def test_is_account_not_authorized_404(self):
         """tests if account_not_authorized method in discover raises http
@@ -73,3 +82,4 @@ def test_is_account_not_authorized_404(self):
 
         expected_error_message = "404 Client Error: Not Found for url:"
         self.assertIn(expected_error_message, str(e.exception))
+

tests/unittests/test_http_exceptions.py

@@ -47,3 +47,4 @@ def test_rate_limit_502_error(self, mocked_send, mocked_sleep):
         with self.assertRaises(RateLimitException):
             client.request("departments")
         self.assertEqual(mocked_send.call_count, 10)
+