From 58c752dce943982d133eebe3a7e499310cf9d1c5 Mon Sep 17 00:00:00 2001 From: Carl Vitzthum Date: Thu, 12 Jul 2018 15:28:50 -0400 Subject: [PATCH 1/4] Some tweaks to existing create_es_client function --- dcicutils/es_utils.py | 48 +++++++++++++++++++++++++------------------ setup.py | 2 +- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/dcicutils/es_utils.py b/dcicutils/es_utils.py index b2ad065e1..0fb5d9abb 100644 --- a/dcicutils/es_utils.py +++ b/dcicutils/es_utils.py @@ -1,5 +1,8 @@ from elasticsearch import Elasticsearch, RequestsHttpConnection -from aws_requests_auth.boto_utils import BotoAWSRequestsAuth +from + + +.boto_utils import BotoAWSRequestsAuth import curator ''' @@ -7,28 +10,23 @@ https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-managedomains-snapshots.html ''' - -def get_index_list(client, name, days_old=0, timestring='%Y.%m.%d', ilo=None): - if ilo is None: - ilo = curator.IndexList(client) - - ilo.filter_by_regex(kind='prefix', value=name) - ilo.filter_by_age(source='name', direction='older', timestring=timestring, unit='days', - unit_count=days_old) - return ilo - - -def create_es_client(es_url, use_aws_auth=False): +def create_es_client(es_url, use_aws_auth=True, **options): + """ + Use to create a ES that supports the signature version 4 signing process. + Need to do role-based IAM access control for AWS hosted ES. + Takes a string es server url, boolean whether or not to use aws auth + signing procedure, and any additional kwargs that will be passed to + creation of the Elasticsearch client. + """ + # may be passed in as a list, as was done previously if isinstance(es_url, (list, tuple)): - addresses = es_url - else: - addresses = [es_url, ] + es_url = es_url[0] es_options = {'retry_on_timeout': True, - 'maxsize': 50 # parallellism... - } + 'maxsize': 50} # parallellism... + es_options.update(**options) # add any given keyword options if use_aws_auth: - host = addresses[0].split('//') + host = es_url.split('//') # remove schema from url host = host[-1].split(":") auth = BotoAWSRequestsAuth(aws_host=host[0].rstrip('/'), aws_region='us-east-1', @@ -36,7 +34,17 @@ def create_es_client(es_url, use_aws_auth=False): es_options['connection_class'] = RequestsHttpConnection es_options['http_auth'] = auth - return Elasticsearch(addresses, **es_options) + return Elasticsearch(es_url, **es_options) + + +def get_index_list(client, name, days_old=0, timestring='%Y.%m.%d', ilo=None): + if ilo is None: + ilo = curator.IndexList(client) + + ilo.filter_by_regex(kind='prefix', value=name) + ilo.filter_by_age(source='name', direction='older', timestring=timestring, unit='days', + unit_count=days_old) + return ilo def create_snapshot_repo(client, repo_name, s3_bucket): diff --git a/setup.py b/setup.py index 59cdc5923..990587c29 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ packages=['dcicutils'], include_package_data=True, zip_safe=False, - author='$dn Team at Harvard Medical School', + author='4DN Team at Harvard Medical School', author_email='jeremy_johnson@hms.harvard.edu', url='https://data.4dnucleome.org', license='MIT', From f37f4d7d59ceb7706c0a306926dbf316f14f5cfe Mon Sep 17 00:00:00 2001 From: Carl Vitzthum Date: Thu, 12 Jul 2018 15:34:03 -0400 Subject: [PATCH 2/4] Typo fix --- dcicutils/es_utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dcicutils/es_utils.py b/dcicutils/es_utils.py index 0fb5d9abb..e46cbe1db 100644 --- a/dcicutils/es_utils.py +++ b/dcicutils/es_utils.py @@ -1,8 +1,5 @@ from elasticsearch import Elasticsearch, RequestsHttpConnection -from - - -.boto_utils import BotoAWSRequestsAuth +from aws_requests_auth.boto_utils import BotoAWSRequestsAuth import curator ''' @@ -24,7 +21,6 @@ def create_es_client(es_url, use_aws_auth=True, **options): es_options = {'retry_on_timeout': True, 'maxsize': 50} # parallellism... - es_options.update(**options) # add any given keyword options if use_aws_auth: host = es_url.split('//') # remove schema from url host = host[-1].split(":") @@ -33,6 +29,7 @@ def create_es_client(es_url, use_aws_auth=True, **options): aws_service='es') es_options['connection_class'] = RequestsHttpConnection es_options['http_auth'] = auth + es_options.update(**options) # add any given keyword options at the end return Elasticsearch(es_url, **es_options) From 6d585de3e52b6166e24ad72b2729414178370236 Mon Sep 17 00:00:00 2001 From: Carl Vitzthum Date: Thu, 12 Jul 2018 16:26:56 -0400 Subject: [PATCH 3/4] Updated version number --- dcicutils/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcicutils/_version.py b/dcicutils/_version.py index 940f292c3..1582477c3 100644 --- a/dcicutils/_version.py +++ b/dcicutils/_version.py @@ -1,4 +1,4 @@ """Version information.""" # The following line *must* be the last in the module, exactly as formatted: -__version__ = "0.3.1" +__version__ = "0.3.2" From 340c445763edc5a2cfc683a0aa69d41f9e346bd7 Mon Sep 17 00:00:00 2001 From: Carl Vitzthum Date: Thu, 12 Jul 2018 16:44:08 -0400 Subject: [PATCH 4/4] Fix flake complaint and updated repo email --- dcicutils/es_utils.py | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dcicutils/es_utils.py b/dcicutils/es_utils.py index e46cbe1db..e986aeae5 100644 --- a/dcicutils/es_utils.py +++ b/dcicutils/es_utils.py @@ -7,6 +7,7 @@ https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-managedomains-snapshots.html ''' + def create_es_client(es_url, use_aws_auth=True, **options): """ Use to create a ES that supports the signature version 4 signing process. diff --git a/setup.py b/setup.py index 990587c29..47624aa13 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ include_package_data=True, zip_safe=False, author='4DN Team at Harvard Medical School', - author_email='jeremy_johnson@hms.harvard.edu', + author_email='burak_alver@hms.harvard.edu', url='https://data.4dnucleome.org', license='MIT', install_requires=requires,