diff --git a/crawler/plugins/emitters/sas_emitter.py b/crawler/plugins/emitters/sas_emitter.py index feb4d3ef..c700af33 100644 --- a/crawler/plugins/emitters/sas_emitter.py +++ b/crawler/plugins/emitters/sas_emitter.py @@ -2,6 +2,7 @@ import os import json import time +import sys import requests @@ -102,7 +103,6 @@ def gen_params(self, namespace='', features='', timestamp='', # remove "icp/" string from namespace namespace = namespace[4:] assert namespace[0] != "/" - logger.info("emit frame (namespace=%s)", namespace) params.update({'namespace': namespace}) params.update({'access_group': access_group}) @@ -162,6 +162,17 @@ def post(self, content='', metadata={}): import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + # skip it in live-crawler if namespace includes reg-crawler format + if ':' in params.get('namespace') \ + and params.get('source_type') == 'container': + logger.info("frame does not satisfy SAS required format") + logger.info("source_type=container, namespace=%s", + params.get('namespace')) + return + + logger.info("emit frame (namespace=%s)", params.get('namespace')) + logger.info("content size: {0} byte".format(sys.getsizeof(content))) + # set interval to avoid burst emit if int(self.emit_interval) > 0: logger.debug("wait %s sec...", self.emit_interval)