diff --git a/.github/workflows/on_push_testing.yml b/.github/workflows/on_push_testing.yml index 60a75bd8..68a4ab14 100644 --- a/.github/workflows/on_push_testing.yml +++ b/.github/workflows/on_push_testing.yml @@ -6,51 +6,85 @@ jobs: lint: runs-on: ubuntu-latest + container: + image: centos:8 steps: - uses: actions/checkout@v1 + - name: Install EPEL + run: dnf install -y epel-release - name: Install dependencies - run: sudo apt-get install python python-pip python-virtualenv python-m2crypto + run: dnf install -y python3 python3-m2crypto - name: Setup virtualenv run: | - virtualenv --system-site-packages venv + python3 -m venv --system-site-packages venv (. venv/bin/activate && pip install --upgrade pip wheel setuptools) - name: Install pdm from push - run: (. venv/bin/activate && pip install --user -e .) + run: (. venv/bin/activate && pip install -e .) - name: Install Tox and dependencies (for testing) run: (. venv/bin/activate && pip install tox) - name: Run tox testing - run: (. venv/bin/activate && tox -e lint test) + run: (. venv/bin/activate && tox -e lint test) + - name: Upload output artifact + if: failure() + uses: actions/upload-artifact@v1 + with: + name: lint_output + path: .tox/lint unit: runs-on: ubuntu-latest + container: + image: centos:8 steps: - uses: actions/checkout@v1 + - name: Install EPEL + run: dnf install -y epel-release - name: Install dependencies - run: sudo apt-get install python python-pip python-virtualenv python-m2crypto + run: dnf install -y python3 python3-m2crypto - name: Setup virtualenv run: | - virtualenv --system-site-packages venv + python3 -m venv --system-site-packages venv (. venv/bin/activate && pip install --upgrade pip wheel setuptools) - name: Install pdm from push - run: (. venv/bin/activate && pip install --user -e .) + run: (. venv/bin/activate && pip install -e .) - name: Install Tox and dependencies (for testing) run: (. venv/bin/activate && pip install tox) - name: Run tox testing run: (. venv/bin/activate && tox -e unit test) + - name: Upload output artifact + if: failure() + uses: actions/upload-artifact@v1 + with: + name: unit_output + path: .tox/unit full: runs-on: ubuntu-latest + container: + image: centos:8 steps: - uses: actions/checkout@v1 + - name: Install EPEL + run: dnf install -y epel-release - name: Install dependencies - run: sudo apt-get install python python-pip python-virtualenv python-m2crypto + run: dnf install -y openssl python3 python3-m2crypto python3-pyOpenSSL - name: Setup virtualenv run: | - virtualenv --system-site-packages venv + python3 -m venv --system-site-packages venv (. venv/bin/activate && pip install --upgrade pip wheel setuptools) - name: Install pdm from push - run: (. venv/bin/activate && pip install --user -e .) + run: (. venv/bin/activate && pip install -e .) +##Fixes ERROR: virtualenv 20.0.3 has requirement six<2,>=1.12.0, but you'll have six 1.11.0 which is incompatible. +## This virtualenv is used by the tox (3.14.3) system. +## The old version of six must come from the distribution (system-site-package) of python3-pyOpenSSL which is only +## installed for full testing - name: Install Tox and dependencies (for testing) - run: (. venv/bin/activate && pip install tox) + run: (. venv/bin/activate && pip install 'six>=1.12.0,<2' tox==3.14.3) - name: Run tox testing run: (. venv/bin/activate && tox -e full test) + - name: Upload output artifact + if: failure() + uses: actions/upload-artifact@v1 + with: + name: full_output + path: .tox/full diff --git a/bin/test_demo.py b/bin/test_demo.py index b829e367..e03a33ff 100755 --- a/bin/test_demo.py +++ b/bin/test_demo.py @@ -11,14 +11,14 @@ def show_turtles(client): turtles = client.get_turtles() - print "Server Turtles: %s" % ', '.join(turtles.values()) + print("Server Turtles: %s" % ', '.join(turtles.values())) def main(): return_code = 0 if len(sys.argv) != 2: - print "Usage: test_demo.py " + print("Usage: test_demo.py ") sys.exit(1) # We load a config so that the client can find the server @@ -28,50 +28,50 @@ def main(): os.chdir(os.path.dirname(conf_file)) # Create a client and run the hello function client = DemoClient() - print "Hello Returned: %s" % client.hello().strip() + print("Hello Returned: %s" % client.hello().strip()) # Now do Turtle stuff show_turtles(client) - print "Adding Turtle..." + print("Adding Turtle...") my_id = client.add_turtle("New Turtle")['id'] - print "New turtle is ID: %u" % my_id + print("New turtle is ID: %u" % my_id) show_turtles(client) - print "Modifying turtle with ID %u..." % my_id + print("Modifying turtle with ID %u..." % my_id) mod_turtle = client.modify_turtle(my_id, "My Lovely Turtle") - print "Modified turtle: %s" % mod_turtle + print("Modified turtle: %s" % mod_turtle) show_turtles(client) - print "Deleting new turtle.""" + print("Deleting new turtle.""") client.del_turtle(my_id) show_turtles(client) # Do Token Demo - print "\n\nTOKEN DEMO\n" + print("\n\nTOKEN DEMO\n") token = client.get('get_token') - print "Got Token: %s" % token + print("Got Token: %s" % token) # Try a request without a token try: res = client.get('verify_token') - print "Tokenless request accepted!!! %s" % res + print("Tokenless request accepted!!! %s" % res) return_code += 1 except: - print "No token request rejected as expected. :-)" + print("No token request rejected as expected. :-)") # Try a request with the wrong token try: client.set_token('WRONGTOKEN') res = client.get('verify_token') - print "Invalid token accepted!!! %s" % res + print("Invalid token accepted!!! %s" % res) return_code += 1 except: - print "Wrong token rejected, as expected. :-)" + print("Wrong token rejected, as expected. :-)") # Try a request with the correct token client.set_token(token) res = client.get('verify_token') - print "Verify real token: %s" % res + print("Verify real token: %s" % res) sys.exit(return_code) diff --git a/setup.py b/setup.py index 32c6caa5..dc0bc74b 100644 --- a/setup.py +++ b/setup.py @@ -8,14 +8,14 @@ package_dir={'': 'src'}, packages=find_packages('src'), install_requires=['m2crypto', - 'flask==0.10.1', - 'flask-sqlalchemy==2.0', + 'flask', + 'flask-sqlalchemy', 'pyOpenSSL', # ==0.13.1 on RHEL7 - 'requests==2.6.0', - 'sqlalchemy==0.9.8', - 'twisted==12.1.0', - 'enum34==1.0.4', - 'python-dateutil==1.5'], + 'requests', + 'sqlalchemy', + 'twisted', + 'python-dateutil', + 'service_identity'], # See commit notes: Needed by Twisted. include_package_data=True, scripts=['src/pdm/bin/pdm'], ) diff --git a/src/pdm/CLI/user_subcommand.py b/src/pdm/CLI/user_subcommand.py index 3a00c1ab..772aa9c7 100644 --- a/src/pdm/CLI/user_subcommand.py +++ b/src/pdm/CLI/user_subcommand.py @@ -250,7 +250,7 @@ def not_implemented(self, args): :return: """ - print " Operation not implemented yet ..." + print(" Operation not implemented yet ...") def register(self, args): # pylint: disable=no-self-use """ @@ -261,22 +261,22 @@ def register(self, args): # pylint: disable=no-self-use :return: None """ if not args.name: - args.name = raw_input("Please enter your given name: ") + args.name = input("Please enter your given name: ") if not args.surname: - args.surname = raw_input("Please enter your surname: ") + args.surname = input("Please enter your surname: ") password = getpass() conf_pass = getpass(prompt='Confirm password: ') if password != conf_pass: - print "Passwords don't match. Aborted" + print("Passwords don't match. Aborted") return client = HRClient() userdict = {'surname': args.surname, 'name': args.name, 'email': args.email, 'password': password} client.add_user(userdict) - print "User registered %s %s %s " % (args.name, args.surname, args.email) - print "Verification email sent to %s. Please check your mailbox." % args.email + print("User registered %s %s %s " % (args.name, args.surname, args.email)) + print("Verification email sent to %s. Please check your mailbox." % args.email) def verify(self, args): """ @@ -285,19 +285,19 @@ def verify(self, args): :param args: parser arguments :return: None """ - print "Verifying email address.\nCut and paste the token received by email." - token = raw_input("Token:") + print("Verifying email address.\nCut and paste the token received by email.") + token = input("Token:") token = os.path.basename(token) client = HRClient() tokendict = {'mailtoken': token} try: client.verify_user(tokendict) - print "User email %s verified" % HRUtils.get_token_username_insecure(token) + print("User email %s verified" % HRUtils.get_token_username_insecure(token)) except RESTException as re: if re.code == 400: - print "Token expired or already verified." + print("Token expired or already verified.") else: - print re + print(re) def resend_email(self, args): """ @@ -306,11 +306,11 @@ def resend_email(self, args): :param args: parser arguments (email address) :return: None """ - print "Sending a verification email to %s " % args.email + print("Sending a verification email to %s " % args.email) client = HRClient() userdict = {'email': args.email} client.resend_email(userdict) - print "Email sent. Please check your mailbox." + print("Email sent. Please check your mailbox.") def unregister(self, args): """ @@ -326,9 +326,9 @@ def unregister(self, args): client.set_token(token) client.del_user() os.remove(os.path.expanduser(args.token)) - print "User unregistered. Token deleted." + print("User unregistered. Token deleted.") else: - print "Unregister operation failed." + print("Unregister operation failed.") def login(self, args): # pylint: disable=no-self-use """ @@ -346,14 +346,14 @@ def login(self, args): # pylint: disable=no-self-use password = getpass(prompt=str(username) + "'s password: ") except ValueError as ve: # corrupted or empty token - print ve.message + print(ve) if not password: # username from the command line if not args.email: - args.email = raw_input("Please enter your email address: ") + args.email = input("Please enter your email address: ") if not args.email: - print "No email provided. Exiting .." + print("No email provided. Exiting ..") exit(1) username = args.email password = getpass() @@ -366,14 +366,14 @@ def login(self, args): # pylint: disable=no-self-use os.makedirs(os.path.dirname(filename)) except OSError as exc: if exc.errno != errno.EEXIST: - print os.strerror(exc.errno) + print(os.strerror(exc.errno)) raise with open(filename, "w") as token_file: os.chmod(filename, 0o600) token_file.write(token) - print "User {} logged in".format(username) + print("User {} logged in".format(username)) def logoff(self, args): """ @@ -389,12 +389,12 @@ def logoff(self, args): try: username = HRUtils.get_token_username_insecure(token) except ValueError as ve: - print ve.message + print(ve) finally: os.remove(os.path.expanduser(args.token)) - print "Token deleted, user {} is now logged off".format(str(username)) + print("Token deleted, user {} is now logged off".format(str(username))) else: - print "No token found, no one to log off." + print("No token found, no one to log off.") def passwd(self, args): # pylint: disable=no-self-use """ Change user password. """ @@ -406,13 +406,13 @@ def passwd(self, args): # pylint: disable=no-self-use newpassword1 = getpass(prompt='Confirm New Password: ') if newpassword != newpassword1: - print "Passwords don't match. Aborted" + print("Passwords don't match. Aborted") return client = HRClient() client.set_token(token) ret = client.change_password(password, newpassword) - print ret + print(ret) def whoami(self, args): # pylint: disable=no-self-use """ @@ -442,7 +442,7 @@ def list(self, args): # pylint: disable=no-self-use if token and self._session_ok(args.site, token): client = TransferClientFacade(token) # remove None values, position args, func and token from the kwargs: - accepted_args = {key: value for (key, value) in vars(args).iteritems() if + accepted_args = {key: value for (key, value) in vars(args).items() if value is not None and key not in ('func', 'site', 'token', 'config', 'verbosity', 'wait')} resp = client.list(args.site, **accepted_args) # max_tries, priority, depth) @@ -459,15 +459,15 @@ def list(self, args): # pylint: disable=no-self-use if status['status'] == 'DONE': listing_output = client.output(resp['id'], 0, -1)[0][0] # listing is 0, last attempt listing_d_value = listing_output['listing'] - root, listing = listing_d_value.items()[0] # top root + root, listing = list(listing_d_value.items())[0] # top root self._print_formatted_listing(root, listing_d_value) elif resp['status'] == 'FAILED': - print " Failed to obtain a listing for job %d " % (resp['id'],) + print(" Failed to obtain a listing for job %d " % (resp['id'],)) else: - print "Timeout. Last status is %s for job id %d" % \ - (status['status'], resp['id']) + print("Timeout. Last status is %s for job id %d" % \ + (status['status'], resp['id'])) elif isinstance(resp, list) and not resp: - print "No such site: %s " % (args.site,) + print("No such site: %s " % (args.site,)) def sitelist(self, args): # pylint disable-no-self-use """ @@ -480,12 +480,12 @@ def sitelist(self, args): # pylint disable-no-self-use if token: client = TransferClientFacade(token) sites = client.list_sites() - print '-' + 91 * '-' + '-' - print '|{0:40}|{1:50}|'.format('site:', 'description:') - print '|' + 91 * '-' + '|' + print('-' + 91 * '-' + '-') + print('|{0:40}|{1:50}|'.format('site:', 'description:')) + print('|' + 91 * '-' + '|') for elem in sites: - print '|{site_name:40s}|{site_desc:50s}|'.format(**elem) - print '-' + 91 * '-' + '-' + print('|{site_name:40s}|{site_desc:50s}|'.format(**elem)) + print('-' + 91 * '-' + '-') def _print_formatted_listing(self, root, full_listing, level=0): # pylint: disable=no-self-use """ @@ -500,7 +500,7 @@ def _print_formatted_listing(self, root, full_listing, level=0): # pylint: disa listing = full_listing[root] if not listing: - print "" + print("") return size_len = len(str(max(d['st_size'] for d in listing))) @@ -513,13 +513,13 @@ def _print_formatted_listing(self, root, full_listing, level=0): # pylint: disa for elem in listing: # filter ot bits we don't want: - filtered_elem = {key: value for (key, value) in elem.iteritems() if + filtered_elem = {key: value for (key, value) in elem.items() if value is not None and key not in ('st_atime', 'st_ctime', 'st_ino', 'st_dev')} - print level * indent * ' ', fmt. \ + print(level * indent * ' ', fmt. \ format(**dict(filtered_elem, st_mode=filemode(elem['st_mode']), - st_mtime=str(datetime.utcfromtimestamp(elem['st_mtime'])))) + st_mtime=str(datetime.utcfromtimestamp(elem['st_mtime']))))) if stat.S_ISDIR(elem['st_mode']): if os.path.join(root, elem['name']) in full_listing: @@ -561,17 +561,17 @@ def _status(self, job_id, client, element_id=None, block=False): status = client.status(job_id, element_id) self.__count += 1 if self.__count >= self.__max_iter: - print "Timeout .." + print("Timeout ..") break - print "(%2d) job id: %d status: %s " % (self.__count, job_id, status['status']) + print("(%2d) job id: %d status: %s " % (self.__count, job_id, status['status'])) if element_id is None: - print "Job id: %d status: %s " % (job_id, status['status']) + print("Job id: %d status: %s " % (job_id, status['status'])) else: - print "Job id: %d.%d status: %s " % (job_id, element_id, status['status']) - print "\tattempts: {attempts} transferred[MB]: {transferred}\n\t" \ + print("Job id: %d.%d status: %s " % (job_id, element_id, status['status'])) + print("\tattempts: {attempts} transferred[MB]: {transferred}\n\t" \ "instant[kB/s]: {instant} average[kB/s]:" \ - " {average} elapsed[s]: {elapsed}".format(**status) + " {average} elapsed[s]: {elapsed}".format(**status)) return status def remove(self, args): # pylint: disable=no-self-use @@ -586,7 +586,7 @@ def remove(self, args): # pylint: disable=no-self-use if token and self._session_ok(args.site, token): client = TransferClientFacade(token) # remove None values, position args, func and token from the kwargs: - accepted_args = {key: value for (key, value) in vars(args).iteritems() if + accepted_args = {key: value for (key, value) in vars(args).items() if value is not None and key not in ('func', 'site', 'token', 'block', 'config', 'verbosity')} response = client.remove(args.site, **accepted_args) # max_tries, priority) @@ -609,7 +609,7 @@ def copy(self, args): # pylint: disable=no-self-use src_site = args.src_site dst_site = args.dst_site # remove None values, position args, func and token from the kwargs: - accepted_args = {key: value for (key, value) in vars(args).iteritems() if + accepted_args = {key: value for (key, value) in vars(args).items() if value is not None and key not in ('func', 'src_site', 'dst_site', 'token', 'block', 'config', 'verbosity')} @@ -628,7 +628,7 @@ def mkdir(self, args): token = UserCommand._get_token(args.token) if token and self._session_ok(args.site, token): client = TransferClientFacade(token) - accepted_args = {key: value for (key, value) in vars(args).iteritems() if + accepted_args = {key: value for (key, value) in vars(args).items() if value is not None and key not in ('func', 'site', 'token', 'block', 'config', 'verbosity')} response = client.mkdir(args.site, **accepted_args) # max_tries, priority @@ -647,7 +647,7 @@ def rename(self, args): token = UserCommand._get_token(args.token) if token and self._session_ok(args.oldname, token): client = TransferClientFacade(token) - accepted_args = {key: value for (key, value) in vars(args).iteritems() if + accepted_args = {key: value for (key, value) in vars(args).items() if value is not None and key not in ('func', 'token', 'block', 'config', 'verbosity', 'oldname', 'newname')} @@ -676,9 +676,9 @@ def log(self, args): if args.verbosity == logging.DEBUG: pprint(attempt) else: - print log_listing + print(log_listing) except RESTException as rexc: - print str(rexc) + print(str(rexc)) def jobs(self, args): """ @@ -712,9 +712,9 @@ def get_site(self, args): session_info = site_client.get_session_info(site_id) UserCommand._print_formatted_session_info(session_info) except RESTException as res_ex: - print str(res_ex) + print(str(res_ex)) else: - print "site %s not found !" % (args.name,) + print("site %s not found !" % (args.name,)) def get_session(self, args): """ @@ -731,7 +731,7 @@ def get_session(self, args): session_info = site_client.get_session_info(site_id) UserCommand._print_formatted_session_info(session_info, attach=False) else: - print "site %s not found !" % (args.name,) + print("site %s not found !" % (args.name,)) def _session_ok(self, site_path, token): """ @@ -743,16 +743,16 @@ def _session_ok(self, site_path, token): """ name, path = TransferClientFacade.split_site_path(site_path) if name is None: - print "Malformed site path (should be sitename:path)" + print("Malformed site path (should be sitename:path)") return None site_client, site_id = UserCommand._get_site_id(name, token) ok = False if site_id: ok = site_client.get_session_info(site_id)['ok'] if not ok: - print "Please log to the site %s first" % (name) + print("Please log to the site %s first" % (name)) else: - print "site %s not found !" % (name) + print("site %s not found !" % (name)) return ok def add_site(self, args): @@ -764,7 +764,7 @@ def add_site(self, args): """ token = UserCommand._get_token(args.token) if token: - site_info = {key: value for (key, value) in vars(args).iteritems() if + site_info = {key: value for (key, value) in vars(args).items() if value is not None and key not in ('func', 'token', 'config', 'verbosity', 'service_ca_cert', 'user_ca_cert')} @@ -780,7 +780,7 @@ def add_site(self, args): site_info['service_ca_cert'] = service_cert else: return None - print site_info + print(site_info) site_client = SiteClient() site_client.set_token(token) site_client.add_site(site_info) @@ -799,7 +799,7 @@ def del_site(self, args): if site_id: site_client.del_site(site_id) else: - print "site %s not found !" % (args.name,) + print("site %s not found !" % (args.name,)) def site_login(self, args): """ @@ -819,7 +819,7 @@ def site_login(self, args): else: user = session_info.get('username') if not user: - user = raw_input("Please enter username for site {}:".format(args.name)) + user = input("Please enter username for site {}:".format(args.name)) password = getpass("User [{}], " "please enter password for site {}:".format(user, args.name)) @@ -827,12 +827,12 @@ def site_login(self, args): try: site_client.logon(site_id, user, password, lifetime=args.lifetime, voms=args.voms) - print " user %s logged in at site %s (valid for %d hours)" \ - % (user, args.name, args.lifetime) + print(" user %s logged in at site %s (valid for %d hours)" \ + % (user, args.name, args.lifetime)) except RESTException as res_ex: - print str(res_ex) + print(str(res_ex)) else: - print "site %s not found !" % (args.name,) + print("site %s not found !" % (args.name,)) def site_logoff(self, args): """ @@ -847,11 +847,11 @@ def site_logoff(self, args): if site_id: try: site_client.logoff(site_id) - print "Logged out from site %s" % (args.name,) + print("Logged out from site %s" % (args.name,)) except RESTException as res_ex: - print str(res_ex) + print(str(res_ex)) else: - print "site %s not found !" % (args.name,) + print("site %s not found !" % (args.name,)) @staticmethod def _get_site_name(site_id, token): @@ -883,46 +883,46 @@ def _get_site_id(name, token): @staticmethod def _print_formatted_siteinfo(siteinfo): if not siteinfo: - print " Nothing to print" + print(" Nothing to print") return - print '-' + 91 * '-' + '-' - print '|{0:20}|{1:70}|'.format('site property:', 'value:') - print '|' + 91 * '-' + '|' - for key, value in siteinfo.iteritems(): + print('-' + 91 * '-' + '-') + print('|{0:20}|{1:70}|'.format('site property:', 'value:')) + print('|' + 91 * '-' + '|') + for key, value in siteinfo.items(): if key.endswith('cert'): continue if isinstance(value, list): for item in value: - print '|{0:20}|{1:70}|'.format(key, str(item)) + print('|{0:20}|{1:70}|'.format(key, str(item))) key = ' ' else: - print '|{0:20}|{1:70}|'.format(key, str(value)) - print '-' + 91 * '-' + '-' + print('|{0:20}|{1:70}|'.format(key, str(value))) + print('-' + 91 * '-' + '-') @staticmethod def _print_formatted_session_info(usersession, attach=True): if not usersession: - print " Nothing to print" + print(" Nothing to print") return if not attach: - print '-' + 91 * '-' + '-' - print '|{0:20}|{1:70}|'.format('user session:', 'value:') - print '|' + 91 * '-' + '|' - for key, value in usersession.iteritems(): - print '|{0:20}|{1:70}|'.format(key, str(value)) - print '-' + 91 * '-' + '-' + print('-' + 91 * '-' + '-') + print('|{0:20}|{1:70}|'.format('user session:', 'value:')) + print('|' + 91 * '-' + '|') + for key, value in usersession.items(): + print('|{0:20}|{1:70}|'.format(key, str(value))) + print('-' + 91 * '-' + '-') @staticmethod def _print_formatted_user_info(userinfo): if not userinfo: - print " Nothing to print" + print(" Nothing to print") return - print '-' + 91 * '-' + '-' - print '|{0:20}|{1:70}|'.format('user property:', 'value:') - print '|' + 91 * '-' + '|' - for key, value in userinfo.iteritems(): - print '|{0:20}|{1:70}|'.format(key, str(value)) - print '-' + 91 * '-' + '-' + print('-' + 91 * '-' + '-') + print('|{0:20}|{1:70}|'.format('user property:', 'value:')) + print('|' + 91 * '-' + '|') + for key, value in userinfo.items(): + print('|{0:20}|{1:70}|'.format(key, str(value))) + print('-' + 91 * '-' + '-') @staticmethod def _print_formatted_jobs_info(jobs, token, long_listing=True): @@ -944,9 +944,9 @@ def _print_formatted_jobs_info(jobs, token, long_listing=True): fmt += '{%s:^%d}|' % elem fmth += '{%d:^%d}|' % (i, elem[1]) nchars += elem[1] - print nchars * '-' - print fmth.format(*zip(*keys)[0]) - print nchars * '-' + print(nchars * '-') + print(fmth.format(*zip(*keys)[0])) + print(nchars * '-') sites = {} for job in jobs: @@ -970,13 +970,13 @@ def _print_formatted_jobs_info(jobs, token, long_listing=True): dst_filepath = UserCommand._trim_string_to_size(job['dst_filepath'], dict(keys)['dst_filepath']) - print fmt.format( + print(fmt.format( **dict(job, timestamp=job['timestamp'][:19], src_filepath=src_filepath, dst_filepath=dst_filepath, src_sitename=src_sitename, - dst_sitename=dst_sitename)) - print nchars * '-' + dst_sitename=dst_sitename))) + print(nchars * '-') @staticmethod def _trim_string_to_size(source, size, dots=True): @@ -1011,15 +1011,15 @@ def _get_token(tokenfile, check_validity=True): with open(os.path.expanduser(tokenfile)) as token_file: token = token_file.read() if not token: - print "No token at requested location. Please login first." + print("No token at requested location. Please login first.") return None if check_validity: if HRUtils.is_token_expired_insecure(token): - print "Token expired. Please log in again." + print("Token expired. Please log in again.") return None return token if check_validity: - print "No token at requested location. Please login first." + print("No token at requested location. Please login first.") return None @staticmethod @@ -1028,7 +1028,7 @@ def _get_cert(certfile): with open(os.path.expanduser(certfile)) as cert_file: cert = cert_file.read() if not cert: - print "No certificate at requested location. Please check and try again." + print("No certificate at requested location. Please check and try again.") return cert - print "%s does not exist or it is not a file. Please check and try again." % (certfile,) + print("%s does not exist or it is not a file. Please check and try again." % (certfile,)) return None diff --git a/src/pdm/framework/ACLManager.py b/src/pdm/framework/ACLManager.py index f0ba5fd6..18e5d76f 100644 --- a/src/pdm/framework/ACLManager.py +++ b/src/pdm/framework/ACLManager.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ Access Control for Flask Wrapper. """ -import urllib +import urllib.parse from copy import deepcopy from datetime import datetime import flask @@ -205,7 +205,7 @@ def __match_path(req_detail, rule_detail): return False # Now check each segment of the path to match either # directly or by wildcard - for part_num in xrange(0, len(rule_parts)): + for part_num in range(0, len(rule_parts)): req_part = req_parts[part_num] rule_part = rule_parts[part_num] if rule_part == '?': @@ -229,7 +229,7 @@ def __do_abort(): if ep_func: redir_url = getattr(ep_func, 'export_redir', None) if redir_url: - orig_path = urllib.quote(request.path, safe='') + orig_path = urllib.parse.quote(request.path, safe='') real_redir = redir_url % {'return_to': orig_path} abort(flask.redirect(real_redir)) abort(403) @@ -255,7 +255,7 @@ def __check_acl(self): self.__do_abort() # No specific rule for this path, try generic rules did_match = False - for rule_path in self.__rules.iterkeys(): + for rule_path in self.__rules.keys(): if self.__match_path(real_path, rule_path): did_match = True if self.__matches_rules(self.__rules[rule_path]): diff --git a/src/pdm/framework/FlaskWrapper.py b/src/pdm/framework/FlaskWrapper.py index 20ba462c..29779464 100644 --- a/src/pdm/framework/FlaskWrapper.py +++ b/src/pdm/framework/FlaskWrapper.py @@ -7,6 +7,7 @@ import json import uuid import logging +import collections from pdm.framework.Tokens import TokenService from pdm.framework.ACLManager import ACLManager @@ -116,8 +117,8 @@ def __add_tables(self): """ self.__db.tables = DBContainer() #pylint: disable=protected-access - registry = self.__db.Model._decl_class_registry - for tbl_name, tbl_inst in registry.iteritems(): + registry = self.__db.Model.registry._class_registry + for tbl_name, tbl_inst in registry.items(): if hasattr(tbl_inst, '__tablename__'): setattr(self.__db.tables, tbl_name, tbl_inst) @@ -140,7 +141,7 @@ def __init__(self, server_name, logger=logging.getLogger(), self.__test_funcs = [] self.__logger = logger if not token_key: - token_key = os.urandom(16) + token_key = os.urandom(16).decode('latin1') self.secret_key = token_key + "flask" self.token_svc = TokenService(token_key, "pdmwebsvc") # We override the test client class from Flask with our @@ -212,7 +213,7 @@ def attach_obj(self, obj_inst, root_path='/', if not redir: redir = parent_redir obj_path = os.path.join(root_path, ename) - if not callable(obj_inst): + if not isinstance(obj_inst, collections.Callable): self.__logger.debug("Class %s at %s", obj_inst, obj_path) if hasattr(obj_inst, 'db_model'): self.__logger.debug("Extending DB model: %s", @@ -243,7 +244,7 @@ def add_auth_groups(self, groups): name, value is a list of auth entries. Returns None """ - for group, entries in groups.iteritems(): + for group, entries in groups.items(): for entry in entries: self.__acl_manager.add_group_entry(group, entry) @@ -259,7 +260,7 @@ def add_auth_rules(self, auth_rules): By default no-one can call any function. Returns None. """ - for path, rule in auth_rules.iteritems(): + for path, rule in auth_rules.items(): self.__acl_manager.add_rule(path, rule) def test_mode(self, main_cls, conf="", with_test=True): diff --git a/src/pdm/framework/RESTClient.py b/src/pdm/framework/RESTClient.py index 32664bba..cfecba1c 100644 --- a/src/pdm/framework/RESTClient.py +++ b/src/pdm/framework/RESTClient.py @@ -170,10 +170,10 @@ def patch_client(target, test_client, base_uri='/'): # We import mock here as TestClient is only meant for use in the tests # If we import it globally, it'll break importing this module in # production. - import mock + import unittest.mock # We patch away the base class of the target, replacing it with # RESTClientTest instead. - patcher = mock.patch.object(target, '__bases__', (RESTClientTest, )) + patcher = unittest.mock.patch.object(target, '__bases__', (RESTClientTest, )) patcher.start() # is_local is required to prevent mock from attempting to delete # __bases__ when stop is called (which would throw an exception) diff --git a/src/pdm/framework/Startup.py b/src/pdm/framework/Startup.py index a64e0bb3..625852b7 100644 --- a/src/pdm/framework/Startup.py +++ b/src/pdm/framework/Startup.py @@ -76,7 +76,7 @@ def __init_app(self, app_server, app_name, config): except pydoc.ErrorDuringImport as err: # We failed to import the client app, we need to raise the inner # exception to make debugging easier - raise err.exc, err.value, err.tb + raise err.exc(err.value).with_traceback(err.tb) app_server.build_db() app_server.before_startup(config, with_test=self.__test) # Test if there are any unused keys in the dictionary diff --git a/src/pdm/framework/Tokens.py b/src/pdm/framework/Tokens.py index 6209e00e..ac5097c9 100644 --- a/src/pdm/framework/Tokens.py +++ b/src/pdm/framework/Tokens.py @@ -25,7 +25,7 @@ def set_key(self, key=None): Returns None. """ if not key: - self.__key = os.urandom(32) + self.__key = os.urandom(32).decode('latin1') else: self.__key = key self.__signer = URLSafeSerializer(self.__key, salt=self.__salt) diff --git a/src/pdm/framework/WSGIServer.py b/src/pdm/framework/WSGIServer.py index 975c7d40..9908dc3c 100644 --- a/src/pdm/framework/WSGIServer.py +++ b/src/pdm/framework/WSGIServer.py @@ -3,12 +3,24 @@ """ import os +import logging from twisted.web import server, wsgi #pylint: disable=no-member from twisted.internet import reactor from twisted.internet import ssl + +def _get_x509_attr_or_none(x509name, attr): + try: + ret = getattr(x509name, attr, None) + except: + logger = logging.getLogger(__name__).getChild("_get_x509_attr_or_none") + logger.exception("Error getting attribute %r from x509name object", attr) + ret = None + return ret + + class WSGIAuth(wsgi.WSGIResource): """ This class implements a wrapper around the twisted WSGI module to implement SSL client certificates based authentication. @@ -45,19 +57,20 @@ def __dn_to_str(x509name): """ dn_parts = [] # E-mail is special as it joins on to CN - if hasattr(x509name, 'CN') and x509name.CN: - if hasattr(x509name, 'emailAddress') and x509name.emailAddress: - dn_parts.append("CN=%s/emailAddress=%s" % \ - (x509name.CN, x509name.emailAddress)) + cn = _get_x509_attr_or_none(x509name, 'CN') + email_address = _get_x509_attr_or_none(x509name, 'emailAddress') + if cn is not None: + if email_address is not None: + dn_parts.append("CN=%s/emailAddress=%s" % (cn, email_address)) else: - dn_parts.append("CN=%s" % x509name.CN) - elif hasattr(x509name, 'emailAddress') and x509name.emailAddress: - dn_parts.append("emailAddress=%s" % x509name.emailAddress) + dn_parts.append("CN=%s" % cn) + elif email_address is not None: + dn_parts.append("emailAddress=%s" % email_address) # Now do other, more standard, parts... for field in ('OU', 'O', 'L', 'ST', 'C'): - if not hasattr(x509name, field): + field_val = _get_x509_attr_or_none(x509name, field) + if field_val is None: continue - field_val = getattr(x509name, field) if field_val: dn_parts.append("%s=%s" % (field, field_val)) dn_parts.reverse() diff --git a/src/pdm/installer/install_pdm.py b/src/pdm/installer/install_pdm.py index e0b910b9..5ee99dc6 100755 --- a/src/pdm/installer/install_pdm.py +++ b/src/pdm/installer/install_pdm.py @@ -7,16 +7,16 @@ import sys import socket import getpass -import ConfigParser +import configparser from subprocess import Popen, PIPE try: import requests except ImportError as err: - print "ERROR: Could not find python requests library (%s)." % str(err) - print "Please install this with the standard package manager." - print "On RHEL7 and similar, the required (root) command is:" - print " yum install python-requests" - print "" + print("ERROR: Could not find python requests library (%s)." % str(err)) + print("Please install this with the standard package manager.") + print("On RHEL7 and similar, the required (root) command is:") + print(" yum install python-requests") + print("") sys.exit(1) # Program default options/config @@ -39,32 +39,32 @@ def usage(): """ Print the program usage information and exit. """ - print "Usage: install_pdm.py " - print " install_pdm.py -w" - print "" - print "Installs a PDM client endpoint when provided with a config file." - print "The -w option will write a template config file, 'pdm.conf' in" - print "the current directory." - print "" + print("Usage: install_pdm.py ") + print(" install_pdm.py -w") + print("") + print("Installs a PDM client endpoint when provided with a config file.") + print("The -w option will write a template config file, 'pdm.conf' in") + print("the current directory.") + print("") sys.exit(1) def write_def_config(): """ Write the default config file and exit. """ CONF_FILE = "pdm.conf" if os.path.exists(CONF_FILE): - print "Config file %s already exists! Not overwriting..." % CONF_FILE + print("Config file %s already exists! Not overwriting..." % CONF_FILE) sys.exit(1) with open(CONF_FILE, "w") as conf_fd: conf_fd.write(INSTALLER_CONF % OPTS) - print "Default config written to '%s'." % CONF_FILE + print("Default config written to '%s'." % CONF_FILE) sys.exit(0) def read_config(fname): """ Read the config file 'fname' and update the OPTS dictionary with the new values. """ - print "[*] Loading config file..." - conf = ConfigParser.ConfigParser() + print("[*] Loading config file...") + conf = configparser.ConfigParser() conf.read(fname) # All options are optional at this point for opt in ('conf_dir', 'ca_dir', 'hostname', 'gridftpd_path', @@ -83,12 +83,12 @@ def read_config(fname): if 'hostname' not in OPTS: hostname = socket.gethostname() OPTS['hostname'] = hostname - print " Note: Hostname not set in config. Using: %s" % hostname + print(" Note: Hostname not set in config. Using: %s" % hostname) # Tidy up OPTS['conf_dir'] = os.path.abspath(OPTS['conf_dir']) OPTS['ca_dir'] = os.path.abspath(OPTS['ca_dir']) - print " Using conf dir: %s" % OPTS['conf_dir'] - print " ca dir: %s" % OPTS['ca_dir'] + print(" Using conf dir: %s" % OPTS['conf_dir']) + print(" ca dir: %s" % OPTS['ca_dir']) def find_bin_helper(prog_name, conf_name, include_loc=False): """ Search for prog_name using the following locations: @@ -104,9 +104,9 @@ def find_bin_helper(prog_name, conf_name, include_loc=False): # User set path, only use that if it's there bin_path = OPTS[conf_name] if not os.access(bin_path, os.X_OK): - print "ERROR: Configured path '%s' for '%s' is not found or exectuable." % \ - (bin_path, conf_name) - print "Please review your config file and try again." + print("ERROR: Configured path '%s' for '%s' is not found or exectuable." % \ + (bin_path, conf_name)) + print("Please review your config file and try again.") sys.exit(1) return bin_path # Search the PATH @@ -121,48 +121,48 @@ def find_bin_helper(prog_name, conf_name, include_loc=False): OPTS[conf_name] = bin_path return bin_path # Binary wasn't found - print "ERROR: Failed to find '%s' binary. Please ensure you have all" % prog_name - print "required packages installed or set the path manually in the" - print "config file if it isn't in a standard location." + print("ERROR: Failed to find '%s' binary. Please ensure you have all" % prog_name) + print("required packages installed or set the path manually in the") + print("config file if it isn't in a standard location.") sys.exit(1) def find_bins(): """ Find all of the required binaries on $PATH if they weren't specified in the config file. """ - print "[*] Locating program/daemon files..." + print("[*] Locating program/daemon files...") res = find_bin_helper("globus-gridftp-server", "gridftpd_path") - print " Found globus-gridftp-server at '%s'." % res + print(" Found globus-gridftp-server at '%s'." % res) res = find_bin_helper("myproxy-server", "myproxy_path") - print " Found myproxy-server at '%s'." % res + print(" Found myproxy-server at '%s'." % res) res = find_bin_helper("build_pdm_ca.sh", "ca_builder", True) - print " Found build_pdm_ca.sh helper script at '%s'." % res + print(" Found build_pdm_ca.sh helper script at '%s'." % res) # One last binary to check for: the gridmap myproxy callout library # As this is a library, we'll check for the config file from the package instead if not os.path.exists("/etc/gridmap_verify_myproxy_callout-gsi_authz.conf"): - print "ERROR: Failed to find the myproxy callout library." - print "Please ensure you have the globus-gridmap-verify-myproxy-callout" - print "package installed." + print("ERROR: Failed to find the myproxy callout library.") + print("Please ensure you have the globus-gridmap-verify-myproxy-callout") + print("package installed.") sys.exit(1) def check_conf(): """ Check that the config has all required options. """ - print "[*] Verifying config options..." + print("[*] Verifying config options...") for opt in ('conf_dir', 'ca_dir', 'hostname', 'gridftpd_path', 'myproxy_path', 'myproxy_port', 'cert_hours', 'gridftp_port', 'gridftp_low', 'gridftp_high'): if not opt in OPTS: - print "ERROR: Config option '%s' was not found." % opt - print "Please review your config file and try again." + print("ERROR: Config option '%s' was not found." % opt) + print("Please review your config file and try again.") sys.exit(1) # Check optional sections if 'service_url' in OPTS: for opt in ('sitename', 'sitedesc', 'public'): if not opt in OPTS: - print "ERROR: service_url was set to enable registration," - print "however option '%s' was missing from the config file." % opt - print "Please review your config file and try again." + print("ERROR: service_url was set to enable registration,") + print("however option '%s' was missing from the config file." % opt) + print("Please review your config file and try again.") sys.exit(1) def get_central_ca(): @@ -174,13 +174,13 @@ def get_central_ca(): warnings.filterwarnings("ignore") if 'service_url' not in OPTS: return # Registration not configured, skip this step - print "[*] Getting central services details..." + print("[*] Getting central services details...") full_url = "%s/service" % OPTS['service_url'] - print " Connecting to central info service: %s" % full_url + print(" Connecting to central info service: %s" % full_url) res = requests.get(full_url, verify=False) if res.status_code != 200: - print "ERROR: Failed to connect to central service (%u)." % res.status_code - print " %s" % res.text + print("ERROR: Failed to connect to central service (%u)." % res.status_code) + print(" %s" % res.text) data = res.json() central_ca = data['central_ca'] # Get the fingerprint of the CA for verification @@ -188,20 +188,20 @@ def get_central_ca(): stdin=PIPE, stdout=PIPE, shell=False) stdout, _ = proc.communicate(central_ca) if proc.returncode: - print "ERROR: Failed to check fingerprint of CA cert." + print("ERROR: Failed to check fingerprint of CA cert.") sys.exit(1) stdout = stdout.strip() - print " The central CA certificate has fingerprint:" - print " %s" % stdout - print "" - print " You should check this with the central admin." + print(" The central CA certificate has fingerprint:") + print(" %s" % stdout) + print("") + print(" You should check this with the central admin.") while True: - user_ip = raw_input(" Is this fingerprint correct? [y/n] ") + user_ip = input(" Is this fingerprint correct? [y/n] ") if user_ip == 'y': break if user_ip == 'n': - print "An incorrect fingerprint indicates an insecure connection." - print "Please contact the central admin for further advice." + print("An incorrect fingerprint indicates an insecure connection.") + print("Please contact the central admin for further advice.") sys.exit(1) ca_file = os.path.join(OPTS['conf_dir'], 'central.pem') with open(ca_file, 'w') as ca_fd: @@ -214,52 +214,52 @@ def login_user(): if 'service_url' not in OPTS: return # We need to get the users URL - print "[*] Preparing to login to central service..." + print("[*] Preparing to login to central service...") full_url = "%s/service" % OPTS['service_url'] # Replace double-slash, but avoid breaking https:// full_url = re.sub("([^:]/)/", "\\1", full_url) res = requests.get(full_url, verify=OPTS['ssl_ca']) data = res.json() if not 'user_ep' in data: - print "ERROR: Central service didn't return a user endpoint." + print("ERROR: Central service didn't return a user endpoint.") sys.exit(1) - print " Please supply your login details for the central service:" - email = raw_input(" E-mail address: ") + print(" Please supply your login details for the central service:") + email = input(" E-mail address: ") passwd = getpass.getpass(" Password: ") user_data = {'email': email, 'passwd': passwd} login_ep = "%s/login" % data['user_ep'] - for _ in xrange(3): + for _ in range(3): res = requests.post(login_ep, verify=OPTS['ssl_ca'], json=user_data) if res.status_code != 200: - print "Login request failed (%u), maybe invalid password?" % \ - res.status_code + print("Login request failed (%u), maybe invalid password?" % \ + res.status_code) continue OPTS['token'] = res.json() return - print "Three login attempts failed. Exiting." + print("Three login attempts failed. Exiting.") sys.exit(1) def create_ca_dir(): """ Create system CA files. """ - print "[*] Creating CA files..." - print " This may take some time, please be patient..." + print("[*] Creating CA files...") + print(" This may take some time, please be patient...") # Just run the CA helper cmd_args = [OPTS['ca_builder'], OPTS['ca_dir'], OPTS['hostname']] proc = Popen(cmd_args, stdout=PIPE, stderr=PIPE, shell=False) stdout, stderr = proc.communicate() if proc.returncode: - print "ERROR: Building CAs failed, stderr:" - print stderr - print "stdout:" - print stdout + print("ERROR: Building CAs failed, stderr:") + print(stderr) + print("stdout:") + print(stdout) sys.exit(1) def install_services(): """ Create the config files for all of the PDM services. """ - print "[*] Creating PDM configuration files..." + print("[*] Creating PDM configuration files...") for fname, ftemp in (('gridftpd.conf', GRIDFTP_CONF), ('myproxy.conf', MYPROXY_CONF), ('myproxy_authz.conf', GRIDFTP_AUTH_CONF)): @@ -268,14 +268,14 @@ def install_services(): with open(full_path, 'w') as conf_fd: conf_fd.write(ftemp % OPTS) except Exception as err: - print "ERROR: Failed to write config '%s' (%s)." % (full_path, str(err)) + print("ERROR: Failed to write config '%s' (%s)." % (full_path, str(err))) sys.exit(1) def register_systemd(): """ Create the systemd config file to start the relevant services at system boot. """ - print "[*] Creating systemd unit files..." + print("[*] Creating systemd unit files...") for fname, ftemp in (('pdm-gridftpd.service', GRIDFTP_SYSTEMD), ('pdm-myproxy.service', MYPROXY_SYSTEMD)): full_path = os.path.join(SYSTEMD_UNIT_PATH, fname) @@ -283,29 +283,29 @@ def register_systemd(): with open(full_path, 'w') as conf_fd: conf_fd.write(ftemp % OPTS) except Exception as err: - print "ERROR: Failed to write unit file '%s' (%s)." % (full_path, str(err)) + print("ERROR: Failed to write unit file '%s' (%s)." % (full_path, str(err))) sys.exit(1) - print " Reloading systemd to pick up new files..." + print(" Reloading systemd to pick up new files...") proc = Popen(['systemctl', 'daemon-reload'], stdout=PIPE, stderr=PIPE, shell=False) proc.communicate() def start_services(): """ Start & check the daemons. """ - print "[*] Starting services..." + print("[*] Starting services...") for service_name in ('pdm-gridftpd.service', 'pdm-myproxy.service'): - print " Starting & enabling %s." % service_name + print(" Starting & enabling %s." % service_name) for cmd in (['systemctl', '-q', 'start', service_name], ['systemctl', '-q', 'enable', service_name]): proc = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=False) stdout, stderr = proc.communicate() if proc.returncode: - print "ERROR: Failed to run '%s':" % " ".join(cmd) - print stdout, stderr - print "For furter details, try running:" - print " systemctl status %s" % service_name - print "and:" - print " journalctl -u %s" % service_name + print("ERROR: Failed to run '%s':" % " ".join(cmd)) + print(stdout, stderr) + print("For furter details, try running:") + print(" systemctl status %s" % service_name) + print("and:") + print(" journalctl -u %s" % service_name) sys.exit(1) def register_service(): @@ -314,7 +314,7 @@ def register_service(): """ if 'service_url' not in OPTS: return # Registration not configured, skip this step - print "[*] Registering with central server..." + print("[*] Registering with central server...") # Load the load CA files user_ca = "" with open(os.path.join(OPTS['ca_dir'], 'user/ca_crt.pem'), "r") as pem_fd: @@ -340,15 +340,15 @@ def register_service(): res = requests.post(reg_url, json=reg_data, headers=reg_hdrs, verify=OPTS['ssl_ca']) if res.status_code != 200: - print "ERROR: Failed to register site centrally:" - print res.text + print("ERROR: Failed to register site centrally:") + print(res.text) sys.exit(1) return def main(): """ Main script entry point. """ if os.getuid() != 0: - print "ERROR: This program required root access." + print("ERROR: This program required root access.") sys.exit(1) if len(sys.argv) != 2: usage() @@ -358,7 +358,7 @@ def main(): write_def_config() conf_file = sys.argv[1] if not os.path.isfile(conf_file): - print "Could not find config file '%s'." % conf_file + print("Could not find config file '%s'." % conf_file) sys.exit(1) read_config(conf_file) find_bins() @@ -374,7 +374,7 @@ def main(): register_systemd() start_services() register_service() - print "Congratulations: Configuration finished successfully." + print("Congratulations: Configuration finished successfully.") # The following are all of the config file templates diff --git a/src/pdm/site/SiteService.py b/src/pdm/site/SiteService.py index 7e48b717..e82e9bd7 100644 --- a/src/pdm/site/SiteService.py +++ b/src/pdm/site/SiteService.py @@ -128,8 +128,8 @@ def get_site(site_id): site = Site.query.filter_by(site_id=site_id).first_or_404() is_owner = (site.site_owner == user_id) if not (site.public or is_owner): - log.warn("User %u failed to get site %u details (no permission).", - user_id, site_id) + log.warning("User %u failed to get site %u details (no permission).", + user_id, site_id) abort(404) # User isn't allowed to see this site log.info("User %u got details of site %u (%s).", user_id, site_id, site.site_name) @@ -173,12 +173,12 @@ def add_site(): site_data[key] = raw_val # Check the auth types if site_data["auth_type"] not in (0, 1): - log.warn("Unable to add site: Invalid auth_type (%s)", - site_data["auth_type"]) + log.warning("Unable to add site: Invalid auth_type (%s)", + site_data["auth_type"]) return "Invalid auth_type.", 400 if not SiteService.check_uri(site_data["auth_uri"]): - log.warn("Unable to add site: Invalid auth_uri (%s)", - site_data["auth_uri"]) + log.warning("Unable to add site: Invalid auth_uri (%s)", + site_data["auth_uri"]) return "Invalid auth_uri.", 400 # Extra fields site_data["site_owner"] = SiteService.get_current_uid() @@ -186,11 +186,11 @@ def add_site(): raw_eps = raw_site_data.get('endpoints', []) for raw_ep in raw_eps: if not SiteService.check_uri(raw_ep): - log.warn("Unable to add site: Bad endpoint (%s)", raw_ep) + log.warning("Unable to add site: Bad endpoint (%s)", raw_ep) return "Invalid endpoint format.", 400 endpoints.extend(raw_eps) except Exception as err: - log.warn("POST data error from client: %s", str(err)) + log.warning("POST data error from client: %s", str(err)) return "Malformed POST data", 400 # Now actually try to create the site new_site = Site(**site_data) @@ -272,8 +272,8 @@ def del_user(user_id): auth_user_id = SiteService.get_current_uid() # Check the user is deleting their own items if auth_user_id != user_id: - log.warn("User %u tried to delete sites belonging to user %u.", - auth_user_id, user_id) + log.warning("User %u tried to delete sites belonging to user %u.", + auth_user_id, user_id) abort(404) sites = Site.query.filter_by(site_owner=auth_user_id).all() num_sites = len(sites) @@ -321,7 +321,7 @@ def logon_session(site_id): user_id = SiteService.get_current_uid() # Decode POST data if not request.data: - log.warn("Missing post data for logon.") + log.warning("Missing post data for logon.") return "Missing POST data", 400 cred_data = json.loads(request.data) username = cred_data.get("username", None) @@ -329,25 +329,25 @@ def logon_session(site_id): lifetime = cred_data.get("lifetime", None) vo_name = cred_data.get("vo", None) if not username or not password or not lifetime: - log.warn("Missing post field in logon.") + log.warning("Missing post field in logon.") return "Required field missing", 400 # Check user can see the site site = Site.query.filter_by(site_id=site_id).first_or_404() is_owner = (site.site_owner == user_id) if not (is_owner or site.public): - log.warn("User %u tried to login to site %u (access denied).", - user_id, site_id) + log.warning("User %u tried to login to site %u (access denied).", + user_id, site_id) abort(404) # This user can't see the requested site # Check the site auth configuration if site.auth_type == 1: # VOMS login if not vo_name: - log.warn("User %u did not specify required VO name for site %u", - user_id, site_id) + log.warning("User %u did not specify required VO name for site %u", + user_id, site_id) return "VO required", 400 if not vo_name in current_app.vo_list: - log.warn("User %u requested unknown VO '%s' for login to site %u.", - user_id, vo_name, site_id) + log.warning("User %u requested unknown VO '%s' for login to site %u.", + user_id, vo_name, site_id) return "Unknown VO name", 400 # Process the different possible CA info combinations ca_info = None diff --git a/src/pdm/userservicedesk/HRService.py b/src/pdm/userservicedesk/HRService.py index acc04573..5a8d24dd 100644 --- a/src/pdm/userservicedesk/HRService.py +++ b/src/pdm/userservicedesk/HRService.py @@ -11,8 +11,7 @@ import datetime import smtplib import socket -from email.MIMEMultipart import MIMEMultipart -from email.MIMEText import MIMEText +from email.message import EmailMessage from enum import IntEnum from sqlalchemy import func @@ -22,7 +21,7 @@ from pdm.utils.hashing import hash_pass, check_hash from pdm.framework.Tokens import TokenService import pdm.userservicedesk.models -from HRUtils import HRUtils +from .HRUtils import HRUtils from pdm.site.SiteClient import SiteClient verif_email_body = \ @@ -589,14 +588,14 @@ def compose_and_send(to_address, mail_token, local_exp): smtp_port = current_app.smtp_server_port smtp_server_pwd = current_app.smtp_server_pwd toaddr = to_address - msg = MIMEMultipart() + msg = EmailMessage() msg['From'] = current_app.mail_display_from msg['To'] = to_address msg['Subject'] = current_app.mail_subject ref = os.path.join(current_app.verification_url, mail_token) body = verif_email_body.format(ref, local_exp.strftime('%c %Z')) - msg.attach(MIMEText(body, 'plain')) + msg.set_content(body) try: server = smtplib.SMTP(smtp_server, smtp_port) @@ -620,9 +619,8 @@ def compose_and_send(to_address, mail_token, local_exp): # will continue w/o login, if optional HRService._check_server_requirements(current_app.smtp_server_login_req, smtp_e) - text = msg.as_string() try: - server.sendmail(fromaddr, toaddr, text) + server.send_message(msg) except smtplib.SMTPException as smtp_e: HRService._logger.error("%s, SMTP sendmail error:", smtp_e) raise RuntimeError(smtp_e) diff --git a/src/pdm/userservicedesk/TransferClientFacade.py b/src/pdm/userservicedesk/TransferClientFacade.py index c8960858..810b68a4 100644 --- a/src/pdm/userservicedesk/TransferClientFacade.py +++ b/src/pdm/userservicedesk/TransferClientFacade.py @@ -26,7 +26,7 @@ def list(self, site, **kwargs): if sitename and path: return super(TransferClientFacade, self).list(sitename, path, **kwargs) else: - print "Malformed site path (format sitename:path)", site + print("Malformed site path (format sitename:path)", site) return None def copy(self, src_site, dst_site, **kwargs): @@ -43,11 +43,11 @@ def copy(self, src_site, dst_site, **kwargs): dst_sitename, dst_path = self.split_site_path(dst_site) if not src_sitename: - print "Malformed site path (probably missing colon)", src_site + print("Malformed site path (probably missing colon)", src_site) return None if not dst_sitename: - print "Malformed site path (probably missing colon)", dst_site + print("Malformed site path (probably missing colon)", dst_site) return None return super(TransferClientFacade, self).copy(src_sitename, src_path, @@ -67,7 +67,7 @@ def remove(self, src_site, **kwargs): if sitename and path: return super(TransferClientFacade, self).remove(sitename, path, **kwargs) else: - print "Malformed site path (probably missing sitename or path)", src_site + print("Malformed site path (probably missing sitename or path)", src_site) return None def mkdir(self, sitepath, **kwargs): @@ -81,7 +81,7 @@ def mkdir(self, sitepath, **kwargs): if sitename and path: return super(TransferClientFacade, self).mkdir(sitename, path, **kwargs) else: - print "Malformed site path (probably missing colon or path element):", sitepath + print("Malformed site path (probably missing colon or path element):", sitepath) return None def rename(self, site_path, newname, **kwargs): @@ -95,13 +95,13 @@ def rename(self, site_path, newname, **kwargs): sitename, path = self.split_site_path(site_path) new_site, newpath = self.split_site_path(newname) if new_site or new_site is None: # we expect '' ! - print "Malformed (new) site path (it has to start with a : " \ - "since source and dest share the same site)", new_site + print("Malformed (new) site path (it has to start with a : " \ + "since source and dest share the same site)", new_site) return None if sitename and path: return super(TransferClientFacade, self).rename(sitename, path, newpath, **kwargs) else: - print "Malformed (old) site path (probably missing sitename or path:)", site_path + print("Malformed (old) site path (probably missing sitename or path:)", site_path) return None @staticmethod diff --git a/src/pdm/utils/X509.py b/src/pdm/utils/X509.py index 66f98c7b..bf545d78 100644 --- a/src/pdm/utils/X509.py +++ b/src/pdm/utils/X509.py @@ -129,7 +129,7 @@ def write_policy(ca_pem, target_file): Returns None. """ - if isinstance(ca_pem, unicode): + if isinstance(ca_pem, str): ca_pem = ca_pem.encode('ascii','ignore') cert = X509.load_cert_string(ca_pem, X509.FORMAT_PEM) ca_raw_dn = X509Utils.x509name_to_str(cert.get_subject()) diff --git a/src/pdm/utils/config.py b/src/pdm/utils/config.py index 380767d2..b54dad05 100644 --- a/src/pdm/utils/config.py +++ b/src/pdm/utils/config.py @@ -3,7 +3,7 @@ import logging from os.path import abspath, realpath, expanduser, expandvars from copy import deepcopy -from ConfigParser import SafeConfigParser +from configparser import ConfigParser from collections import defaultdict from .singleton import singleton @@ -26,7 +26,7 @@ def config(self): @property def sections(self): """Get list of sections.""" - return self._config.keys() + return list(self._config.keys()) def get_section(self, section): """Return a given section.""" @@ -34,18 +34,18 @@ def get_section(self, section): def setup(self, filenames='~/.config/pdm/pdm.conf', ignore_errors=False): """Setup the configuration system.""" - config_parser = SafeConfigParser() + config_parser = ConfigParser() config_parser.optionxform = str - if isinstance(filenames, basestring): + if isinstance(filenames, str): filenames = [filenames] filenames = {abspath(realpath(expanduser(expandvars(filename)))) for filename in filenames} for filename in filenames: try: - with open(filename, 'rb') as config_file: - config_parser.readfp(config_file) + with open(filename, 'r') as config_file: + config_parser.read_file(config_file) self._logger.debug("Read config file: %s", filename) except Exception: self._logger.warning("Failed to read config file: %s", filename) diff --git a/src/pdm/utils/daemon.py b/src/pdm/utils/daemon.py index ea5d921b..48ef2500 100644 --- a/src/pdm/utils/daemon.py +++ b/src/pdm/utils/daemon.py @@ -88,7 +88,7 @@ def __init__(self, pidfile, target, args=None, kwargs=None, **options): @property def pid(self): """Get the pid of the running daemon.""" - with open(self._pidfilename, 'rb') as pidfile: + with open(self._pidfilename, 'r') as pidfile: return pidfile.read() def exit(self): @@ -153,7 +153,7 @@ def start(self): maxfd = 1024 # Close all open file descriptors. - for filed in xrange(maxfd): + for filed in range(maxfd): if filed not in self._extra_fds | {pidfile.fileno}: try: os.close(filed) @@ -182,5 +182,5 @@ def start(self): sys.exit(0) except AlreadyLockedError as err: - sys.stderr.write(err.message) + sys.stderr.write(str(err)) sys.stderr.write('\n') diff --git a/src/pdm/utils/hashing.py b/src/pdm/utils/hashing.py index a9929128..789112a5 100644 --- a/src/pdm/utils/hashing.py +++ b/src/pdm/utils/hashing.py @@ -23,12 +23,14 @@ def hash_pass(password, salt=None): every time. Use check_hash to check an existing hash against a password. """ - if not salt: + if salt is None: salt = get_salt() - hashed_pass = hashlib.pbkdf2_hmac(HASH_ALGO, password, + elif isinstance(salt, str): + salt = salt.encode() + hashed_pass = hashlib.pbkdf2_hmac(HASH_ALGO, password.encode(), salt, HASH_ITER) - hash_str = "$5$%s$%s" % (binascii.hexlify(salt), - binascii.hexlify(hashed_pass)) + hash_str = "$5$%s$%s" % (binascii.hexlify(salt).decode(), + binascii.hexlify(hashed_pass).decode()) return hash_str def check_hash(hash_in, password): @@ -48,6 +50,6 @@ def check_hash(hash_in, password): stored_hash = binascii.unhexlify(hash_parts[3]) except TypeError: raise ValueError("Malfomed base64 in hash input") - new_hash = hashlib.pbkdf2_hmac(HASH_ALGO, password, + new_hash = hashlib.pbkdf2_hmac(HASH_ALGO, password.encode(), salt, HASH_ITER) return new_hash == stored_hash diff --git a/src/pdm/utils/lockfile.py b/src/pdm/utils/lockfile.py index bf41d835..67795ed1 100644 --- a/src/pdm/utils/lockfile.py +++ b/src/pdm/utils/lockfile.py @@ -61,7 +61,7 @@ def update_pid(self): def __enter__(self): """Enter context.""" try: - self._pidfile = open(self._filename, 'a+b') + self._pidfile = open(self._filename, 'a+') except IOError: self._logger.exception("Error opening pidfile %s", self._filename) raise diff --git a/src/pdm/utils/myproxy.py b/src/pdm/utils/myproxy.py index 67036059..4306bf3a 100644 --- a/src/pdm/utils/myproxy.py +++ b/src/pdm/utils/myproxy.py @@ -35,7 +35,7 @@ def logon(myproxy_server, username, password, Returns a string with the new credential PEM. Raises a RuntimeError exception if anything goes wrong. """ - with tempfile.NamedTemporaryFile() as proxy: + with tempfile.NamedTemporaryFile(mode='w') as proxy: hostname, port = myproxy_server.split(':', 1) myproxy_opts = ['myproxy-logon', # Exectuable name '-s', hostname, # MyProxy server name @@ -67,9 +67,9 @@ def logon(myproxy_server, username, password, log.debug("Running myproxy-logon with: %s", " ".join(myproxy_opts)) log.debug(" myproxy-logon env: %s", str(env)) proc = Popen(myproxy_opts, shell=False, stdin=PIPE, stdout=PIPE, - stderr=PIPE, env=env) + stderr=PIPE, env=env, encoding="utf-8") try: - stdout, stderr = proc.communicate('%s\n' % password) + stdout, stderr = proc.communicate(password) except Exception as err: if log: log.warn("myproxy-logon command failed: %s", str(err)) diff --git a/src/pdm/utils/sshkey.py b/src/pdm/utils/sshkey.py index 469ea45d..76293751 100644 --- a/src/pdm/utils/sshkey.py +++ b/src/pdm/utils/sshkey.py @@ -22,12 +22,12 @@ def __rsapub_to_ssh(key_in): """ # It would be nice if a library did this, while some are available # they are all overkill for a relatively simple conversion - algo = 'ssh-rsa' + algo = b'ssh-rsa' algo_hdr = struct.pack('>I', len(algo)) + algo exponent, modulus = key_in.pub() # exponent and modulus already have the length encoded buf = algo_hdr + exponent + modulus - return base64.b64encode(buf) + return base64.b64encode(buf).decode() @staticmethod def gen_rsa_keypair(passphrase=None, comment="", bits=DEF_NUM_BITS): @@ -50,7 +50,7 @@ def gen_rsa_keypair(passphrase=None, comment="", bits=DEF_NUM_BITS): if not passphrase: key_pem = key.as_pem(cipher=None) else: - pw_cb = lambda x: passphrase + pw_cb = lambda x: passphrase if passphrase is None else passphrase.encode() key_pem = key.as_pem(cipher=SSHKeyUtils.DEF_ENC_ALGO, callback=pw_cb) return (pub_str, key_pem) @@ -62,6 +62,6 @@ def remove_pass(key_in, passphrase=None): passphrase - The password of the input key. Returns PEM encoded key with no password. """ - pw_cb = lambda x: passphrase + pw_cb = lambda x: passphrase if passphrase is None else passphrase.encode() key = RSA.load_key_string(key_in, callback=pw_cb) - return key.as_pem(cipher=None) + return key.as_pem(cipher=None).decode() diff --git a/src/pdm/web/WebPageService.py b/src/pdm/web/WebPageService.py index 46ead575..fd564d6f 100644 --- a/src/pdm/web/WebPageService.py +++ b/src/pdm/web/WebPageService.py @@ -393,7 +393,7 @@ def js_list(): if status['status'] == 'DONE': listing_output = [dict(f, is_dir=stat.S_ISDIR(f['st_mode'])) for f in - tclient.output(jobinfo['id'], 0, -1)[0][0]['listing'].values()[0]] + list(tclient.output(jobinfo['id'], 0, -1)[0][0]['listing'].values())[0]] elif jobinfo['status'] == 'FAILED': current_app.log.error("Failed to obtain a listing for job %d", jobinfo['id']) else: diff --git a/src/pdm/workqueue/Worker.py b/src/pdm/workqueue/Worker.py index 78b7b220..9d8d912c 100644 --- a/src/pdm/workqueue/Worker.py +++ b/src/pdm/workqueue/Worker.py @@ -12,12 +12,12 @@ import asyncore import logging import threading -from cStringIO import StringIO +from io import StringIO from collections import defaultdict from pprint import pformat from datetime import datetime from contextlib import contextmanager -from urlparse import urlsplit, urlunsplit +from urllib.parse import urlsplit, urlunsplit from tempfile import NamedTemporaryFile from requests.exceptions import Timeout @@ -72,14 +72,14 @@ def temporary_proxy_files(src_credentials, dst_credentials=None): dict: A dictionary containing the proxy environment variables to set which point to the newly created temporary proxy files. """ - with NamedTemporaryFile() as src_proxyfile: + with NamedTemporaryFile(mode='w') as src_proxyfile: src_proxyfile.write(src_credentials) src_proxyfile.flush() os.fsync(src_proxyfile.fileno()) if dst_credentials is None: yield {'X509_USER_PROXY': src_proxyfile.name} return - with NamedTemporaryFile() as dst_proxyfile: + with NamedTemporaryFile(mode='w') as dst_proxyfile: dst_proxyfile.write(dst_credentials) dst_proxyfile.flush() os.fsync(dst_proxyfile.fileno()) @@ -113,7 +113,7 @@ def writable(self): def handle_read(self): """Handle read events.""" - self._buffer += self.recv(8192) + self._buffer += self.recv(8192).decode() class StdOutDispatcher(asyncore.file_dispatcher): @@ -148,7 +148,7 @@ def force_complete(self, returncode, extra_log=''): 'timestamp': datetime.utcnow().isoformat(), 'host': socket.gethostbyaddr(socket.getfqdn())} stderr = self._stderr_dispatcher.buffer - for element_id, token in self._tokens.iteritems(): + for element_id, token in self._tokens.items(): log = self._log_dict.pop(element_id, StringIO()) log.write(stderr) log.write('\n') @@ -169,7 +169,7 @@ def force_complete(self, returncode, extra_log=''): def handle_read(self): """Handle read events.""" - self._buffer += self.recv(8192) + self._buffer += self.recv(8192).decode() buffered_elements = self._buffer.split('\n') self._buffer = buffered_elements.pop() @@ -216,7 +216,7 @@ def handle_read(self): if 'Listing' in done_element: data['listing'] = {} - for root, listing in done_element['Listing'].iteritems(): + for root, listing in done_element['Listing'].items(): root = urlsplit(root).path if root.startswith('/~'): root = root.lstrip('/') @@ -395,7 +395,7 @@ def run(self): script_env = dict(os.environ, X509_CERT_DIR=ca_dir, **proxy_env_vars) if self._logger.isEnabledFor(logging.DEBUG): extra_env = {key: script_env[key] for key in - set(script_env.iterkeys()).difference(os.environ.iterkeys())} + set(script_env.keys()).difference(iter(os.environ.keys()))} self._logger.debug("Extra environment variables: %s", pformat(extra_env)) self._logger.debug("Sending subprocess the following data: %s", pformat(data)) self._current_process = subprocess.Popen(command, @@ -403,7 +403,8 @@ def run(self): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=script_env) + env=script_env, + encoding="utf-8") json.dump(data, self._current_process.stdin) self._current_process.stdin.write('\n') self._current_process.stdin.flush() diff --git a/src/pdm/workqueue/WorkqueueDB.py b/src/pdm/workqueue/WorkqueueDB.py index 752710f8..038b6dbb 100644 --- a/src/pdm/workqueue/WorkqueueDB.py +++ b/src/pdm/workqueue/WorkqueueDB.py @@ -38,7 +38,7 @@ def parse(cls, obj): """Convert arg to enum.""" if isinstance(obj, int): return cls(obj) - if isinstance(obj, basestring): + if isinstance(obj, str): if obj.isdigit(): return cls(int(obj)) return cls[obj.upper()] @@ -167,7 +167,7 @@ class Job(db_base, JSONMixin, SmartColumnAwareMixin): timestamp = Column(TIMESTAMP, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow) priority = SmartColumn(SmallInteger, - CheckConstraint('priority in {0}'.format(tuple(xrange(10)))), + CheckConstraint('priority in {0}'.format(tuple(range(10)))), nullable=False, allowed=True, default=5) type = SmartColumn(SmallInteger, # pylint: disable=invalid-name CheckConstraint('type in {0}'.format(JobType.values())), diff --git a/src/pdm/workqueue/WorkqueueService.py b/src/pdm/workqueue/WorkqueueService.py index f2bc3ad6..cc7e6e0c 100644 --- a/src/pdm/workqueue/WorkqueueService.py +++ b/src/pdm/workqueue/WorkqueueService.py @@ -231,7 +231,7 @@ def return_output(job_id, element_id): str(element_id)) if not os.path.exists(dir_): os.makedirs(dir_) - with open(os.path.join(dir_, 'attempt%i.log' % element.attempts), 'wb') as logfile: + with open(os.path.join(dir_, 'attempt%i.log' % element.attempts), 'w') as logfile: logfile.write("Job run on host: %s, returncode: %s, timestamp: %s\n" % (request.data['host'], request.data['returncode'], @@ -243,7 +243,7 @@ def return_output(job_id, element_id): and element.type == JobType.LIST\ and element.status == JobStatus.DONE: element_counter = 0 - element_listing = sorted(element.listing.iteritems(), key=lambda item: len(item[0])) + element_listing = sorted(element.listing.items(), key=lambda item: len(item[0])) dir_copy = False if element_listing and \ element_listing[0][0].rstrip('/') == job.src_filepath.rstrip('/'): @@ -270,7 +270,7 @@ def return_output(job_id, element_id): and element.type == JobType.LIST\ and element.status == JobStatus.DONE: element_counter = 0 - for root, listing in sorted(element.listing.iteritems(), + for root, listing in sorted(element.listing.items(), key=lambda item: len(item[0]), reverse=True): for entry in sorted(listing, key=lambda x: stat.S_ISDIR(int(x['st_mode']))): element_counter += 1 @@ -328,13 +328,13 @@ def post_job(): try: job = Job(**request.data) except ValueError as err: - abort(400, description=err.message) + abort(400, description=str(err)) except Exception as err: # pylint: disable=broad-except - abort(500, description=err.message) + abort(500, description=str(err)) try: job.add() except Exception as err: # pylint: disable=broad-except - abort(500, description=err.message) + abort(500, description=str(err)) return jsonify(job) @staticmethod @@ -448,13 +448,13 @@ def get_output(job_id): attempt_list = [] attempts = element.attempts element_log_filebase = os.path.join(log_filebase, str(element.id)) - for attempt in xrange(1, attempts): # previous attempts, presumably failed ones + for attempt in range(1, attempts): # previous attempts, presumably failed ones failed_output = dict(attempt_output, attempt=attempt, status=JobStatus.FAILED.name) log_filename = os.path.join(element_log_filebase, "attempt%i.log" % attempt) log = "log directory/file %s not found for job.element %s.%s."\ % (log_filename, job_id, element.id) if os.path.exists(log_filename): - with open(log_filename, 'rb') as logfile: + with open(log_filename, 'r') as logfile: log = logfile.read() failed_output.update(log=log) attempt_list.append(failed_output) @@ -468,7 +468,7 @@ def get_output(job_id): log = "log directory/file %s not found for job.element %s.%s."\ % (log_filename, job_id, element.id) if os.path.exists(log_filename): - with open(log_filename, 'rb') as logfile: + with open(log_filename, 'r') as logfile: log = logfile.read() last_output.update(log=log) if status == JobStatus.DONE and element.type == JobType.LIST: @@ -510,7 +510,7 @@ def get_element_output(job_id, element_id, attempt=None): # pylint: disable=too abort(400, description="bad attempt, expected an integer.") if attempt < 0: attempt = element.attempts + attempt + 1 - if attempt not in xrange(1, element.attempts + 1): + if attempt not in range(1, element.attempts + 1): abort(404, description="Invalid attempt '%s', job.element %s.%s has been tried %s " "time(s)" % (attempt, job_id, element_id, element.attempts)) @@ -522,7 +522,7 @@ def get_element_output(job_id, element_id, attempt=None): # pylint: disable=too if attempt == element.attempts and element.status == JobStatus.DONE: status = JobStatus.DONE attempt_output.update(attempt=attempt, status=status.name) - with open(log_filename, 'rb') as logfile: + with open(log_filename, 'r') as logfile: attempt_output.update(log=logfile.read()) if status == JobStatus.DONE and element.type == JobType.LIST: attempt_output.update(listing=element.listing) @@ -530,13 +530,13 @@ def get_element_output(job_id, element_id, attempt=None): # pylint: disable=too attempt_list = [] attempts = element.attempts - for attempt_ in xrange(1, attempts): # previous attempts, presumably failed ones + for attempt_ in range(1, attempts): # previous attempts, presumably failed ones failed_output = dict(attempt_output, attempt=attempt_, status=JobStatus.FAILED.name) log_filename = os.path.join(log_filebase, "attempt%i.log" % attempt_) log = "log directory/file %s not found for job.element %s.%s."\ % (log_filename, job_id, element.id) if os.path.exists(log_filename): - with open(log_filename, 'rb') as logfile: + with open(log_filename, 'r') as logfile: log = logfile.read() failed_output.update(log=log) attempt_list.append(failed_output) @@ -550,7 +550,7 @@ def get_element_output(job_id, element_id, attempt=None): # pylint: disable=too log = "log directory/file %s not found for job.element %s.%s."\ % (log_filename, job_id, element.id) if os.path.exists(log_filename): - with open(log_filename, 'rb') as logfile: + with open(log_filename, 'r') as logfile: log = logfile.read() last_output.update(log=log) if status == JobStatus.DONE and element.type == JobType.LIST: diff --git a/src/pdm/workqueue/scripts/pdm_gfal2_copy.py b/src/pdm/workqueue/scripts/pdm_gfal2_copy.py index a20ee16d..9b6bdc16 100755 --- a/src/pdm/workqueue/scripts/pdm_gfal2_copy.py +++ b/src/pdm/workqueue/scripts/pdm_gfal2_copy.py @@ -101,7 +101,7 @@ def pdm_gfal_copy(copy_dict, s_cred_file=None, t_cred_file=None, overwrite=False params.timeout = timeout # unzip: - _, src_l, dst_l = zip(*copy_list) # don't care about jobid + _, src_l, dst_l = list(zip(*copy_list)) # don't care about jobid s_root = str(os.path.dirname(os.path.commonprefix(src_l))) d_root = str(os.path.dirname(os.path.commonprefix(dst_l))) diff --git a/test/pdm/CLI/test_usercommand.py b/test/pdm/CLI/test_usercommand.py index 22595da1..394b1984 100644 --- a/test/pdm/CLI/test_usercommand.py +++ b/test/pdm/CLI/test_usercommand.py @@ -1,5 +1,5 @@ -import mock import unittest +import unittest.mock as mock import argparse import tempfile import datetime @@ -15,7 +15,7 @@ def setUp(self): self._parser = argparse.ArgumentParser() subparsers = self._parser.add_subparsers() UserCommand(subparsers) - self._tmp_file = tempfile.NamedTemporaryFile(dir='/tmp') + self._tmp_file = tempfile.NamedTemporaryFile(dir='/tmp', mode='w') future_date = (datetime.timedelta(0, 600) + datetime.datetime.utcnow()).isoformat() plain = {'id': 44, 'expiry': future_date} svc = TokenService() @@ -241,7 +241,7 @@ def test_add_site(self, mock_site_client): # user provides service certificates, file OK mock_site_client.reset_mock() - _tmp_file = tempfile.NamedTemporaryFile(dir='/tmp') + _tmp_file = tempfile.NamedTemporaryFile(dir='/tmp', mode='w') fake_cert = 'BEGIN_FAKE_CERTghsgshhgkxxxEND_FAKE_CERT' _tmp_file.write(fake_cert) _tmp_file.flush() diff --git a/test/pdm/demo/test_DemoClient.py b/test/pdm/demo/test_DemoClient.py index 0245b4ab..323d87f4 100644 --- a/test/pdm/demo/test_DemoClient.py +++ b/test/pdm/demo/test_DemoClient.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -import mock import unittest +import unittest.mock as mock from pdm.demo.DemoClient import DemoClient from pdm.demo.DemoService import DemoService diff --git a/test/pdm/demo/test_DemoService.py b/test/pdm/demo/test_DemoService.py index 4a53d445..b115fc7a 100644 --- a/test/pdm/demo/test_DemoService.py +++ b/test/pdm/demo/test_DemoService.py @@ -47,7 +47,7 @@ def test_web(self): self.__service.testing = True res = self.__test.get('/web/turtles') assert(res.status_code == 200) - assert("" in res.data) + assert("" in res.get_data(as_text=True)) def test_hello(self): res = self.__test.get('/demo/api/v1.0/hello') @@ -68,7 +68,7 @@ def test_addGetDelTurtle(self): res = self.__test.post('/demo/api/v1.0/turtles', data=new_turtle) assert(res.status_code == 200) turtle_id = json.loads(res.data)['id'] - print "Turtle ID: %s" % turtle_id + print("Turtle ID: %s" % turtle_id) # Get info res = self.__test.get('/demo/api/v1.0/turtles/%u' % turtle_id) assert(res.status_code == 200) @@ -97,11 +97,11 @@ def test_delTimmy(self): assert(res.status_code == 200) turtles = json.loads(res.data) timmy_id = None - for turtle_id, turtle_name in turtles.iteritems(): + for turtle_id, turtle_name in turtles.items(): if turtle_name == 'Timmy': timmy_id = int(turtle_id) break - print "Timmy ID: %s" % timmy_id + print("Timmy ID: %s" % timmy_id) assert(timmy_id is not None) # Found Timmy, now try delete res = self.__test.delete('/demo/api/v1.0/turtles/%u' % timmy_id) @@ -111,7 +111,7 @@ def test_getToken(self): res = self.__test.get('/demo/api/v1.0/get_token') assert(res.status_code == 200) assert(len(res.data) > 10) - assert("." in res.data) + assert("." in res.get_data(as_text=True)) # Actually check token content token_data = self.__service.token_svc.check(json.loads(res.data)) assert(token_data == "Hello") diff --git a/test/pdm/framework/test_ACLManager.py b/test/pdm/framework/test_ACLManager.py index 72625f81..0dab48ce 100644 --- a/test/pdm/framework/test_ACLManager.py +++ b/test/pdm/framework/test_ACLManager.py @@ -140,7 +140,7 @@ def test_auth_modes(self): (False, False, False, False, False, True, False, )), ] for auth_mode, auth_data, auth_res in AUTH_TESTS: - for i in xrange(0, len(TEST_EP)): + for i in range(0, len(TEST_EP)): res = self.__gen_req(TEST_EP[i], "GET", auth_mode, auth_data) self.assertEqual(res, auth_res[i], "Path %s failed, auth_data=%s, Expected: %s, Actual: %s" % \ diff --git a/test/pdm/framework/test_Database.py b/test/pdm/framework/test_Database.py index 9511eb81..c5c05407 100644 --- a/test/pdm/framework/test_Database.py +++ b/test/pdm/framework/test_Database.py @@ -1,10 +1,10 @@ #!/usr/bin/env """ Framework database object tests. """ -import mock import json import datetime import unittest +import unittest.mock as mock from pdm.framework.Database import MemSafeSQLAlchemy from pdm.framework.Database import JSONMixin, JSONTableEncoder @@ -72,7 +72,7 @@ def test_plain(self): test_dict = {'A': 1, 'B': 2, 'C': my_time} json_str = json.dumps(test_dict, cls=JSONTableEncoder) output_dict = json.loads(json_str) - self.assertItemsEqual(test_dict.keys(), output_dict.keys()) + self.assertCountEqual(list(test_dict.keys()), list(output_dict.keys())) self.assertEqual(test_dict['A'], output_dict['A']) self.assertEqual(test_dict['B'], output_dict['B']) self.assertEqual(my_time.isoformat(), output_dict['C']) diff --git a/test/pdm/framework/test_RESTClient.py b/test/pdm/framework/test_RESTClient.py index 7b06165b..d1921f3c 100644 --- a/test/pdm/framework/test_RESTClient.py +++ b/test/pdm/framework/test_RESTClient.py @@ -2,8 +2,8 @@ """ Test RESTClient framework module. """ import json -import mock import unittest +import unittest.mock as mock import functools import requests diff --git a/test/pdm/framework/test_WSGIServer.py b/test/pdm/framework/test_WSGIServer.py index 3b12b392..12a8ae17 100644 --- a/test/pdm/framework/test_WSGIServer.py +++ b/test/pdm/framework/test_WSGIServer.py @@ -1,8 +1,8 @@ #!/usr/bin/env python """ Tests for the WSGIServer module. """ -import mock import unittest +import unittest.mock as mock from pdm.framework.WSGIServer import WSGIAuth, WSGIServer @@ -92,7 +92,7 @@ def test_add_nonssl_server(self, mock_reactor): self.assertEqual(port, TEST_PORT) self.assertEqual(site.resource._application, app_server) - @mock.patch("__builtin__.open") + @mock.patch("builtins.open") @mock.patch("pdm.framework.WSGIServer.ssl") @mock.patch("pdm.framework.WSGIServer.reactor") def test_add_ssl_server(self, mock_reactor, mock_ssl, mock_open): diff --git a/test/pdm/site/test_SiteClient.py b/test/pdm/site/test_SiteClient.py index 85caaa95..5e81b44c 100644 --- a/test/pdm/site/test_SiteClient.py +++ b/test/pdm/site/test_SiteClient.py @@ -1,10 +1,10 @@ #!/usr/bin/env python """ Test SiteService client class. """ -import mock import logging import datetime import unittest +import unittest.mock as mock from pdm.site.SiteClient import SiteClient from pdm.site.SiteService import SiteService diff --git a/test/pdm/site/test_SiteService.py b/test/pdm/site/test_SiteService.py index 9ca40140..b71e3ab4 100644 --- a/test/pdm/site/test_SiteService.py +++ b/test/pdm/site/test_SiteService.py @@ -3,10 +3,10 @@ import copy import json -import mock import logging import datetime import unittest +import unittest.mock as mock from flask import current_app from pdm.site.SiteService import SiteService @@ -267,7 +267,7 @@ def test_vomsdir(self, mp_utils): self.assertEqual(res.status_code, 200) service_info = json.loads(res.data) self.assertIn('vos', service_info) - self.assertItemsEqual(TEST_VOS, service_info['vos']) + self.assertCountEqual(TEST_VOS, service_info['vos']) def test_session_basics(self): """ Manually add a credential to the DB and check that all of the @@ -422,7 +422,7 @@ def test_logon_errors(self, mp_mock): mp_mock.logon.side_effect = Exception(MSG) res = self.__client.post('/site/api/v1.0/session/2', data=AUTH_DATA) self.assertEqual(res.status_code, 400) - self.assertIn(MSG, res.data) + self.assertIn(MSG, res.get_data(as_text=True)) @mock.patch("pdm.site.SiteService.managed_session") @mock.patch("pdm.site.SiteService.X509Utils") diff --git a/test/pdm/userservicedesk/test_HRClient.py b/test/pdm/userservicedesk/test_HRClient.py index aaf2627e..5932cb23 100644 --- a/test/pdm/userservicedesk/test_HRClient.py +++ b/test/pdm/userservicedesk/test_HRClient.py @@ -4,7 +4,7 @@ import json import unittest -import mock +import unittest.mock as mock import datetime from pdm.userservicedesk.HRClient import HRClient @@ -65,7 +65,7 @@ def test_hello(self): def test_login(self): res = self.__client.login('Johnny@example.com', 'very_secret') - assert (isinstance(res, unicode)) + assert (isinstance(res, str)) with self.assertRaises(Exception) as login_ex: res = self.__client.login('Johnny@example.com', 'very_secret1') @@ -93,7 +93,7 @@ def test_change_password(self): self.__service.fake_auth("TOKEN", {'id': 1, 'expiry': self.__future_date}) # client takes plain passwords res = self.__client.change_password('very_secret', 'newpassword') - print res + print(res) assert (res['email'] == self.__userdict['email']) with self.assertRaises(Exception) as pwd_ex: diff --git a/test/pdm/userservicedesk/test_HRService.py b/test/pdm/userservicedesk/test_HRService.py index 66f7e443..698cae6b 100644 --- a/test/pdm/userservicedesk/test_HRService.py +++ b/test/pdm/userservicedesk/test_HRService.py @@ -2,7 +2,7 @@ import os import time import unittest -import mock +import unittest.mock as mock import copy import datetime import smtplib @@ -379,7 +379,7 @@ def test_resend_email(self, smtp_mock): assert res.status_code == 400 - @mock.patch('email.MIMEMultipart.MIMEMultipart') + @mock.patch('email.message.EmailMessage') @mock.patch.object(smtplib.SMTP, 'connect') @mock.patch.object(smtplib.SMTP, 'close') def test_compose_and_send(self, close_mock, connect_mock, mail_mock): @@ -399,7 +399,7 @@ def test_compose_and_send(self, close_mock, connect_mock, mail_mock): HRService.compose_and_send("centos@localhost", 'mytoken_abc', datetime.datetime.utcnow()) - @mock.patch('email.MIMEMultipart.MIMEMultipart') + @mock.patch('email.message.EmailMessage') @mock.patch('smtplib.SMTP') def test_compose_and_send_sendmail(self, smtp_mock, mail_mock): with self.__service.test_request_context(path="/test"): @@ -407,14 +407,12 @@ def test_compose_and_send_sendmail(self, smtp_mock, mail_mock): mytoken = 'mytoken_abc' toaddr = "user@remotehost" body = os.path.join(self._conf['verification_url'], mytoken) - smtp_mock.return_value.sendmail.side_effect = smtplib.SMTPException + smtp_mock.return_value.send_message.side_effect = smtplib.SMTPException with self.assertRaises(RuntimeError): HRService.compose_and_send(toaddr, mytoken, datetime.datetime.utcnow()) - args = smtp_mock.return_value.sendmail.call_args - assert args[0][0] == self._conf['smtp_server_login'] - assert args[0][1] == toaddr - assert body in args[0][2] # check the important part of the email + args = smtp_mock.return_value.send_message.call_args + assert body in args[0][0].get_content() # check the important part of the email def test_hello(self): res = self.__test.get('/users/api/v1.0/hello') diff --git a/test/pdm/userservicedesk/test_TransferClient.py b/test/pdm/userservicedesk/test_TransferClient.py index b93c6cf2..7e34ad90 100644 --- a/test/pdm/userservicedesk/test_TransferClient.py +++ b/test/pdm/userservicedesk/test_TransferClient.py @@ -1,10 +1,9 @@ -import mock import unittest +import unittest.mock as mock import datetime from pdm.userservicedesk.TransferClientFacade import TransferClientFacade from pdm.framework.FlaskWrapper import FlaskServer from pdm.userservicedesk.HRService import HRService -from urlparse import urlparse import pdm.framework.Tokens as Tokens @@ -69,7 +68,7 @@ def test_list(self): assert mock_list.called mock_list.assert_called_with(self.site_id, '/root/file.txt', priority=2) - print mock_list.call_args_list + print(mock_list.call_args_list) wrongurl = "localhost2:/root/file.txt" # no such site, with mock.patch.object(self.__client._TransferClient__wq_client, 'list') as mock_list: @@ -79,10 +78,10 @@ def test_list(self): def test_sitelist(self): sites = self.__client.list_sites() - print sites + print(sites) assert sites[0]['site_name'] == 'localhost' assert sites[1]['site_name'] == 'remotehost' - assert 'site_id' not in [dd.keys() for dd in sites] + assert 'site_id' not in [list(dd.keys()) for dd in sites] def test_remove(self): site = "localhost:/root/file.txt" @@ -93,7 +92,7 @@ def test_remove(self): assert mock_remove.called mock_remove.assert_called_with(self.site_id, '/root/file.txt', priority=2) - print mock_remove.call_args_remove + print(mock_remove.call_args_remove) wrongurl = "localhost2:/root/file.txt" # no such site, with mock.patch.object(self.__client._TransferClient__wq_client, 'remove') as mock_remove: diff --git a/test/pdm/utils/test_X509.py b/test/pdm/utils/test_X509.py index 874f0417..d9c158a7 100644 --- a/test/pdm/utils/test_X509.py +++ b/test/pdm/utils/test_X509.py @@ -1,8 +1,8 @@ #!/usr/bin/env python import os -import mock import unittest +import unittest.mock as mock from functools import partial from pdm.utils.X509 import X509Utils diff --git a/test/pdm/utils/test_config.py b/test/pdm/utils/test_config.py index 0064e438..ca95b2a6 100644 --- a/test/pdm/utils/test_config.py +++ b/test/pdm/utils/test_config.py @@ -30,7 +30,7 @@ def test_config(self): def test_get_section_list(self): """Test get section list.""" self.cfg._config = self.test_ref - self.assertEqual(self.cfg.sections, self.test_ref.keys()) + self.assertEqual(self.cfg.sections, list(self.test_ref.keys())) def test_get_section(self): """Test getting a section.""" @@ -44,7 +44,7 @@ def test_getConfig(self): # pylint: disable=invalid-name def test_setup(self): """Test the setup method.""" - with NamedTemporaryFile() as tmpfile: + with NamedTemporaryFile(mode='w') as tmpfile: tmpfile.write(dedent(""" [my_section] my_var_str = 'hello world' @@ -61,7 +61,7 @@ def test_setup(self): self.cfg.setup('blah') # Test corrupted config. - with NamedTemporaryFile() as tmpfile: + with NamedTemporaryFile(mode='w') as tmpfile: tmpfile.write("blah ][") tmpfile.flush() with self.assertRaises(Exception): @@ -72,8 +72,8 @@ def test_setup(self): def test_multi_configs(self): """Test reading in multiple configs.""" - with NamedTemporaryFile() as file1,\ - NamedTemporaryFile() as file2: + with NamedTemporaryFile(mode='w') as file1,\ + NamedTemporaryFile(mode='w') as file2: file1.write(dedent(""" [section_one] var_one = 12 diff --git a/test/pdm/utils/test_db.py b/test/pdm/utils/test_db.py index b8e0edfd..9a7bc2d5 100644 --- a/test/pdm/utils/test_db.py +++ b/test/pdm/utils/test_db.py @@ -1,8 +1,8 @@ #!/usr/bin/env python """ Test DB utils. """ -import mock import unittest +import unittest.mock as mock from pdm.utils.db import managed_session @@ -11,7 +11,7 @@ class TestDBUtils(unittest.TestCase): def run_db_session(self, test_request, **kwargs): with managed_session(test_request, **kwargs) as db_session: - self.assertEquals(db_session, test_request.db.session) + self.assertEqual(db_session, test_request.db.session) def test_managed_session(self): """ Test managed_session generator. """ diff --git a/test/pdm/utils/test_lockfile.py b/test/pdm/utils/test_lockfile.py index 3eaea736..0a11cd0f 100644 --- a/test/pdm/utils/test_lockfile.py +++ b/test/pdm/utils/test_lockfile.py @@ -2,9 +2,9 @@ """ Test of utils lockfile module. """ import os -import mock import fcntl import unittest +import unittest.mock as mock from pdm.utils.lockfile import PIDLockFile, AlreadyLockedError @@ -14,7 +14,7 @@ class TestPIDLockFile(unittest.TestCase): @mock.patch("logging.getLogger") @mock.patch("os.fsync") @mock.patch("fcntl.flock") - @mock.patch("__builtin__.open") + @mock.patch("builtins.open") def test_locking(self, open_fcn, flock_fcn, fsync_fcn, logging_fcn): """ Check that the locking works as expected. """ @@ -30,7 +30,7 @@ def test_locking(self, open_fcn, flock_fcn, with pid_lock: # The file should now be "locked" self.assertEqual(pid_lock.fileno, 123) - open_fcn.assert_called_with(TEST_FILENAME, "a+b") + open_fcn.assert_called_with(TEST_FILENAME, "a+") flock_fcn.assert_called_with(123, fcntl.LOCK_EX | fcntl.LOCK_NB) self.assertFalse(file_handle.close.called) self.assertTrue(file_handle.close.called) diff --git a/test/pdm/utils/test_myproxy.py b/test/pdm/utils/test_myproxy.py index 2b637dd0..9b074f77 100644 --- a/test/pdm/utils/test_myproxy.py +++ b/test/pdm/utils/test_myproxy.py @@ -1,8 +1,8 @@ #!/usr/bin/env python import os -import mock import unittest +import unittest.mock as mock from pdm.utils.myproxy import MyProxyUtils @@ -37,7 +37,7 @@ def test_myproxy_logon(self, popen_mock, x509_mock, open_mock): # Simplest test res = MyProxyUtils.logon("localhost:12345", "user", "pass") self.assertEqual(res, 'PEMFILE') - self.assertEqual(proc.communicate.call_args[0][0], 'pass\n') + self.assertEqual(proc.communicate.call_args[0][0], 'pass') args, _ = self.get_args(popen_mock) self.assertIn('-s localhost', args) self.assertIn('-p 12345', args) @@ -121,5 +121,5 @@ def test_load_voms_list(self, os_mock, open_mock): open_mock.side_effect = self.voms_open_fcn res = MyProxyUtils.load_voms_list("/mydir") self.assertIsInstance(res, list) - self.assertItemsEqual(res, ["vo1", "vo2.test.vo"]) + self.assertCountEqual(res, ["vo1", "vo2.test.vo"]) self.assertTrue(sorted(res)) diff --git a/test/pdm/utils/test_sshkey.py b/test/pdm/utils/test_sshkey.py index 34a7dacd..eac4b020 100644 --- a/test/pdm/utils/test_sshkey.py +++ b/test/pdm/utils/test_sshkey.py @@ -33,13 +33,13 @@ def test_enc_gen_key(self): comment = pubkey.split(' ', 2)[2] self.assertEqual(comment, TEST_COMMENT) # Check the key is encrypted - pw_cb = lambda x: None + pw_cb = lambda x: b'' # None not working! raises SystemError within M2Crypto self.assertRaises(RSA.RSAError, RSA.load_key_string, privkey, callback=pw_cb) - pw_cb = lambda x: "wrongpass" + pw_cb = lambda x: b"wrongpass" self.assertRaises(RSA.RSAError, RSA.load_key_string, privkey, callback=pw_cb) - pw_cb = lambda x: TEST_PASS + pw_cb = lambda x: TEST_PASS.encode() key_obj = RSA.load_key_string(privkey, callback=pw_cb) self.assertIsInstance(key_obj, RSA.RSA) @@ -49,12 +49,12 @@ def test_remove_pass(self): # Generate a test key with a password key = RSA.gen_key(2048, 5, callback=lambda: None) key_pem = key.as_pem(cipher='aes_256_cbc', - callback=lambda x: TEST_PASS) + callback=lambda x: TEST_PASS.encode()) # Now try to decrypt the key with the helper function key_out = SSHKeyUtils.remove_pass(key_pem, TEST_PASS) # Check returned key looks OK and is password-less self.assertIn('BEGIN RSA PRIVATE KEY', key_out) - key_back_in = RSA.load_key_string(key_out, + key_back_in = RSA.load_key_string(key_out.encode(), callback=lambda x: None) # Finally, test with wrong password self.assertRaises(RSA.RSAError, SSHKeyUtils.remove_pass, diff --git a/test/pdm/web/test_WebPageService.py b/test/pdm/web/test_WebPageService.py index 6d916602..1ae223a9 100644 --- a/test/pdm/web/test_WebPageService.py +++ b/test/pdm/web/test_WebPageService.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import unittest -import mock +import unittest.mock as mock from pdm.web.WebPageService import WebPageService from pdm.framework.FlaskWrapper import FlaskServer @@ -28,14 +28,14 @@ def test_web(self): self.assertIn('/web/datamover', res.location) res = self.__test.get('/web/datamover') self.assertEqual(res.status_code, 200) - self.assertIn("", res.data) + self.assertIn("", res.get_data(as_text=True)) def test_about(self): """ Check that the about page is returned at the correct location. """ res = self.__test.get('/static/about.html') # check its existence self.assertEqual(res.status_code, 200) - self.assertIn("About the datamover", res.data) + self.assertIn("About the datamover", res.get_data(as_text=True)) diff --git a/test/pdm/workqueue/scripts/test_stdout_dump_helper.py b/test/pdm/workqueue/scripts/test_stdout_dump_helper.py index 70394913..2a3c1c68 100644 --- a/test/pdm/workqueue/scripts/test_stdout_dump_helper.py +++ b/test/pdm/workqueue/scripts/test_stdout_dump_helper.py @@ -2,7 +2,7 @@ import unittest import logging -import mock +import unittest.mock as mock from pdm.workqueue.scripts import stdout_dump_helper diff --git a/test/pdm/workqueue/test_Worker.py b/test/pdm/workqueue/test_Worker.py index c65920b3..39632c5a 100644 --- a/test/pdm/workqueue/test_Worker.py +++ b/test/pdm/workqueue/test_Worker.py @@ -2,7 +2,7 @@ """ Test Worker module. """ import logging import unittest -import mock +import unittest.mock as mock from pdm.framework.FlaskWrapper import FlaskServer, jsonify from pdm.framework.RESTClient import RESTClientTest diff --git a/test/pdm/workqueue/test_WorkqueueClient.py b/test/pdm/workqueue/test_WorkqueueClient.py index 624cbcb8..f4345815 100644 --- a/test/pdm/workqueue/test_WorkqueueClient.py +++ b/test/pdm/workqueue/test_WorkqueueClient.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ Test WorkqueueClient module. """ import unittest -import mock +import unittest.mock as mock from pdm.framework.FlaskWrapper import FlaskServer, jsonify from pdm.framework.RESTClient import RESTClientTest diff --git a/test/pdm/workqueue/test_WorkqueueService.py b/test/pdm/workqueue/test_WorkqueueService.py index d3729965..9e459944 100644 --- a/test/pdm/workqueue/test_WorkqueueService.py +++ b/test/pdm/workqueue/test_WorkqueueService.py @@ -3,7 +3,7 @@ import json import unittest from textwrap import dedent -import mock +import unittest.mock as mock from pdm.framework.FlaskWrapper import FlaskServer from pdm.workqueue.WorkqueueDB import JobType, JobStatus, JobProtocol @@ -25,7 +25,7 @@ def setUp(self): src_filepath='/data/somefile1', type=JobType.LIST)) job = Job(user_id=2, src_siteid=14, src_filepath='/data/somefile2', type=JobType.REMOVE) - for i in xrange(1, 6): + for i in range(1, 6): job.elements.append(JobElement(id=i, job_id=2, src_siteid=12, src_filepath='/data/somefile2.%d' % i, type=JobType.REMOVE, size=10**i)) @@ -34,7 +34,7 @@ def setUp(self): src_siteid=15, src_filepath='/data/somefile3', dst_siteid=16, dst_filepath='/data/newfile') - for i in xrange(1, 6): + for i in range(1, 6): j.elements.append(JobElement(id=i, job_id=3, src_siteid=12, src_filepath='/data/somefile3.%d' % i, dst_filepath='/data/newfile.%d' % i, @@ -97,7 +97,7 @@ def setUp(self): src_filepath='/data/somefile1', type=JobType.LIST)) job = Job(user_id=2, src_siteid=14, src_filepath='/data/somefile2', type=JobType.REMOVE) - for i in xrange(1, 6): + for i in range(1, 6): job.elements.append(JobElement(id=i, job_id=2, src_siteid=12, src_filepath='/data/somefile2.%d' % i, type=JobType.REMOVE, size=10**i)) @@ -106,7 +106,7 @@ def setUp(self): src_siteid=15, src_filepath='/data/somefile3', dst_siteid=16, dst_filepath='/data/newfile') - for i in xrange(1, 6): + for i in range(1, 6): j.elements.append(JobElement(id=i, job_id=3, src_siteid=12, src_filepath='/data/somefile3.%d' % i, dst_filepath='/data/newfile.%d' % i, @@ -117,6 +117,16 @@ def setUp(self): self.__service.before_startup(conf) # to continue startup self.__test = self.__service.test_client() + def _check_dict_subset(self, first, second, allow_identical=False): + # Check all keys in first present in second + if allow_identical: + self.assertLessEqual(set(first), set(second)) + else: + self.assertLess(set(first), set(second)) + # Get matching subset from second and check values + self.assertEqual(first, {key: second[key] for key in first}) + + def test_get_next_job(self): """test worker get next job.""" request = self.__test.post('/workqueue/api/v1.0/worker/jobs', data={'test': 12}) @@ -127,31 +137,31 @@ def test_get_next_job(self): self.assertEqual(len(work), 1) job = work[0] self.assertEqual(len(job['elements']), 1) - self.assertDictContainsSubset({'status': JobStatus.SUBMITTED, - 'dst_credentials': None, - 'user_id': 1, - 'src_filepath': '/data/somefile1', - 'priority': 5, - 'dst_siteid': None, - 'src_siteid': 13, - 'extra_opts': None, - 'protocol': 0, - 'type': JobType.LIST, - 'id': 1, - 'src_credentials': None, - 'dst_filepath': None}, job)#, "Job not returned correctly.") + self._check_dict_subset({'status': JobStatus.SUBMITTED, + 'dst_credentials': None, + 'user_id': 1, + 'src_filepath': '/data/somefile1', + 'priority': 5, + 'dst_siteid': None, + 'src_siteid': 13, + 'extra_opts': None, + 'protocol': 0, + 'type': JobType.LIST, + 'id': 1, + 'src_credentials': None, + 'dst_filepath': None}, job) element = job['elements'][0] - self.assertIsInstance(element['token'], basestring) - self.assertDictContainsSubset({'status': JobStatus.SUBMITTED, - 'job_id': 1, - 'attempts': 0, - 'src_filepath': '/data/somefile1', - 'listing': None, - 'max_tries': 2, - 'type': JobType.LIST, - 'id': 0, - 'dst_filepath': None}, element) + self.assertIsInstance(element['token'], str) + self._check_dict_subset({'status': JobStatus.SUBMITTED, + 'job_id': 1, + 'attempts': 0, + 'src_filepath': '/data/somefile1', + 'listing': None, + 'max_tries': 2, + 'type': JobType.LIST, + 'id': 0, + 'dst_filepath': None}, element) Job = self.__service.test_db().tables.Job JobElement = self.__service.test_db().tables.JobElement #j = Job.query.filter_by(id=job['id']).one() @@ -169,7 +179,7 @@ def test_get_next_job(self): self.assertEqual(work[0]['type'], JobType.REMOVE) self.assertEqual(len(work[0]['elements']), 6) self.assertEqual(work[0]['elements'][0]['type'], JobType.LIST) - for i in xrange(1, 6): + for i in range(1, 6): self.assertEqual(work[0]['elements'][i]['type'], JobType.REMOVE) self.assertEqual(work[1]['type'], JobType.COPY) self.assertEqual(len(work[1]['elements']), 6) @@ -235,7 +245,7 @@ def test_return_output(self): Job run on host: somehost.domain, returncode: 1, timestamp: timestamp blah blah """).strip() - with open(logfile, 'rb') as log: + with open(logfile, 'r') as log: self.assertEqual(log.read(), expected_log) request = self.__test.put('/workqueue/api/v1.0/worker/jobs/1/elements/0', @@ -258,7 +268,7 @@ def test_return_output(self): Job run on host: somehost.domain, returncode: 0, timestamp: timestamp blah blah """).strip() - with open(logfile, 'rb') as log: + with open(logfile, 'r') as log: self.assertEqual(log.read(), expected_log) self.assertEqual(je.listing, {'root': []}) @@ -461,7 +471,7 @@ def test_post_job(self, mock_hrservice, mock_siteclient): self.assertEqual(job.src_credentials, 'somesecret') self.assertEqual(job.protocol, JobProtocol.GRIDFTP) self.assertEqual(returned_job['protocol'], 'GRIDFTP') - self.assertIsInstance(job.log_uid, basestring) + self.assertIsInstance(job.log_uid, str) self.assertEqual(len(job.elements), 1) element = job.elements[0] self.assertEqual(element.type, JobType.LIST) @@ -508,7 +518,7 @@ def test_post_job(self, mock_hrservice, mock_siteclient): self.assertEqual(job.dst_credentials, "someothersecret") self.assertEqual(job.protocol, JobProtocol.SSH) self.assertEqual(returned_job['protocol'], 'SSH') - self.assertIsInstance(job.log_uid, basestring) + self.assertIsInstance(job.log_uid, str) self.assertNotEqual(job.log_uid, 'my_log_uid') self.assertEqual(len(job.elements), 1) element = job.elements[0] @@ -597,9 +607,9 @@ def test_get_jobs(self, mock_hrservice): self.assertEqual(request.status_code, 200) returned_jobs = json.loads(request.data) self.assertEqual(len(returned_jobs), 1) - self.assertDictContainsSubset({'user_id': 1, - 'type': 'LIST', - 'status': 'NEW'}, returned_jobs[0]) + self._check_dict_subset({'user_id': 1, + 'type': 'LIST', + 'status': 'NEW'}, returned_jobs[0]) @mock.patch('pdm.userservicedesk.HRService.HRService.check_token') def test_get_job(self, mock_hrservice): @@ -615,9 +625,9 @@ def test_get_job(self, mock_hrservice): request = self.__test.get('/workqueue/api/v1.0/jobs/2') self.assertEqual(request.status_code, 200) returned_job = json.loads(request.data) - self.assertDictContainsSubset({'user_id': 2, - 'type': 'REMOVE', - 'status': 'NEW'}, returned_job) + self._check_dict_subset({'user_id': 2, + 'type': 'REMOVE', + 'status': 'NEW'}, returned_job) @mock.patch('pdm.userservicedesk.HRService.HRService.check_token') def test_get_elements(self, mock_hrservice): @@ -632,9 +642,9 @@ def test_get_elements(self, mock_hrservice): self.assertEqual(request.status_code, 200) returned_elements = json.loads(request.data) self.assertEqual(len(returned_elements), 1) - self.assertDictContainsSubset({'job_id': 1, - 'type': 'LIST', - 'status': 'NEW'}, returned_elements[0]) + self._check_dict_subset({'job_id': 1, + 'type': 'LIST', + 'status': 'NEW'}, returned_elements[0]) @mock.patch('pdm.userservicedesk.HRService.HRService.check_token') def test_get_element(self, mock_hrservice): @@ -654,9 +664,9 @@ def test_get_element(self, mock_hrservice): request = self.__test.get('/workqueue/api/v1.0/jobs/2/elements/0') self.assertEqual(request.status_code, 200) returned_element = json.loads(request.data) - self.assertDictContainsSubset({'job_id': 2, - 'type': 'LIST', - 'status': 'NEW'}, returned_element) + self._check_dict_subset({'job_id': 2, + 'type': 'LIST', + 'status': 'NEW'}, returned_element) @mock.patch('pdm.userservicedesk.HRService.HRService.check_token') def test_get_job_status(self, mock_hrservice): @@ -777,9 +787,9 @@ def test_get_output(self, mock_hrservice): os.makedirs(remove_job0_dir) if not os.path.exists(remove_job1_dir): os.makedirs(remove_job1_dir) - with open(list_job_filename, 'wb') as listlog,\ - open(remove_job0_filename, 'wb') as removelog0,\ - open(remove_job1_filename, 'wb') as removelog1: + with open(list_job_filename, 'w') as listlog,\ + open(remove_job0_filename, 'w') as removelog0,\ + open(remove_job1_filename, 'w') as removelog1: listlog.write('la la la\n') removelog0.write('blah blah\n') removelog1.write('tralala\n') diff --git a/tox.ini b/tox.ini index c9f778c8..c4ab1622 100644 --- a/tox.ini +++ b/tox.ini @@ -3,16 +3,15 @@ envlist = unit [testenv] # Inherited by other test environments -basepython=python2.7 +basepython=python3.6 # Required to allow pyOpenSSL from host on EL7 sitepackages=True # Standard unit testing + coverage [testenv:unit] deps= - mock==1.0.1 - pytest==2.7.0 - coverage==3.6 + pytest + coverage commands= coverage erase coverage run --source pdm -m py.test {posargs} @@ -26,7 +25,6 @@ commands= [testenv:lint] deps= - mock==1.0.1 - pylint==1.6.5 + pylint commands= /bin/bash test/bin/run_pylint.sh