diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28d3fabf..8f1b8f05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -185,6 +185,41 @@ jobs: # gmpy doesn't build with 3.12 # coverage to codeclimate can be submitted just once opt-deps: ['m2crypto', 'gmpy2', 'codeclimate'] + - name: py2.7 with brotli + os: ubuntu-20.04 + python-version: 2.7 + # zstandard is available for py3.8 and above + opt-deps: ['brotli'] + - name: py3.6 with brotli + os: ubuntu-20.04 + python-version: 3.6 + # zstandard is available for py3.8 and above + opt-deps: ['brotli'] + - name: py3.7 with brotli + os: ubuntu-latest + python-version: 3.7 + # zstandard is available for py3.8 and above + opt-deps: ['brotli'] + - name: py3.8 with brotli and zstandard + os: ubuntu-latest + python-version: 3.8 + opt-deps: ['brotli', 'zstd'] + - name: py3.9 with brotli and zstandard + os: ubuntu-latest + python-version: 3.9 + opt-deps: ['brotli', 'zstd'] + - name: py3.10 with brotli and zstandard + os: ubuntu-latest + python-version: '3.10' + opt-deps: ['brotli', 'zstd'] + - name: py3.11 with brotli and zstandard + os: ubuntu-latest + python-version: '3.11' + opt-deps: ['brotli', 'zstd'] + - name: py3.12with brotli and zstandard + os: ubuntu-latest + python-version: '3.12' + opt-deps: ['brotli', 'zstd'] steps: - uses: actions/checkout@v2 if: ${{ !matrix.container }} @@ -300,6 +335,17 @@ jobs: if: ${{ contains(matrix.opt-deps, 'gmpy2') && matrix.python-version == '3.12' }} # for py3.12 we need pre-release version: https://github.com/aleaxit/gmpy/issues/446 run: pip install --pre gmpy2 + - name: Install brotli for Python 2 + if: ${{ contains(matrix.opt-deps, 'brotli') && matrix.python-version == '2.7' }} + # using 1.0.9 for Python 2 because latest is not compatible + # https://github.com/google/brotli/issues/1074 + run: pip install brotli==1.0.9 + - name: Install brotli + if: ${{ contains(matrix.opt-deps, 'brotli') && matrix.python-version != '2.7' }} + run: pip install brotli + - name: Install zstandard for py3.8 and after + if: ${{ contains(matrix.opt-deps, 'zstd') && matrix.python-version != '2.7' && matrix.python-version != '3.6' && matrix.python-version != '3.7' }} + run: pip install zstandard - name: Install build dependencies (2.6) if: ${{ matrix.python-version == '2.6' }} run: | @@ -310,7 +356,7 @@ jobs: wget https://files.pythonhosted.org/packages/72/20/7f0f433060a962200b7272b8c12ba90ef5b903e218174301d0abfd523813/unittest2-1.1.0-py2.py3-none-any.whl wget https://files.pythonhosted.org/packages/85/d5/818d0e603685c4a613d56f065a721013e942088047ff1027a632948bdae6/coverage-4.5.4.tar.gz wget https://files.pythonhosted.org/packages/a8/5a/5cf074e1c6681dcbb4e640113f58bed16955e7da9a6c8090b518031775e7/hypothesis-2.0.0.tar.gz - wget https://files.pythonhosted.org/packages/f8/86/410d53faff049641f34951843245d168261512aea787a1f9f05c3fa025a0/pylint-1.7.6-py2.py3-none-any.whl + wget https://files.pythonhosted.org/packages/f8/86/410d53faff049641f34951843245d168261512aea787a1f9f05c3fa025a0/pylint-1.7.6-py2.py3-none-any.whl wget https://files.pythonhosted.org/packages/81/a6/d076eeb83f383ac7a25e030709abebc6781bcf930d67316be6d47641637e/diff_cover-4.0.0-py2.py3-none-any.whl wget https://files.pythonhosted.org/packages/8c/2d/aad7f16146f4197a11f8e91fb81df177adcc2073d36a17b1491fd09df6ed/pycparser-2.18.tar.gz wget https://files.pythonhosted.org/packages/4b/2a/0276479a4b3caeb8a8c1af2f8e4355746a97fab05a372e4a2c6a6b876165/idna-2.7-py2.py3-none-any.whl @@ -336,7 +382,7 @@ jobs: wget https://files.pythonhosted.org/packages/c2/f8/49697181b1651d8347d24c095ce46c7346c37335ddc7d255833e7cde674d/ipaddress-1.0.23-py2.py3-none-any.whl wget https://files.pythonhosted.org/packages/c7/a3/c5da2a44c85bfbb6eebcfc1dde24933f8704441b98fdde6528f4831757a6/linecache2-1.0.0-py2.py3-none-any.whl wget https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl - wget https://files.pythonhosted.org/packages/bd/c9/6fdd990019071a4a32a5e7cb78a1d92c53851ef4f56f62a3486e6a7d8ffb/urllib3-1.23-py2.py3-none-any.whl + wget https://files.pythonhosted.org/packages/bd/c9/6fdd990019071a4a32a5e7cb78a1d92c53851ef4f56f62a3486e6a7d8ffb/urllib3-1.23-py2.py3-none-any.whl wget https://files.pythonhosted.org/packages/5e/a0/5f06e1e1d463903cf0c0eebeb751791119ed7a4b3737fdc9a77f1cdfb51f/certifi-2020.12.5-py2.py3-none-any.whl wget https://files.pythonhosted.org/packages/8d/08/00aab975c99d156aec2d47e9e7a947ac3af3efab5065f666c8b157acc7a8/lazy_object_proxy-1.3.1-cp26-cp26mu-manylinux1_x86_64.whl wget https://files.pythonhosted.org/packages/82/f7/e43cefbe88c5fd371f4cf0cf5eb3feccd07515af9fd6cf7dbf1d1793a797/wrapt-1.12.1.tar.gz diff --git a/LICENSE b/LICENSE index d29479ce..bb75fde8 100644 --- a/LICENSE +++ b/LICENSE @@ -3,7 +3,7 @@ TLS Lite includes code from different sources. All code is either dedicated to the public domain by its authors, available under a BSD-style license or available under GNU LGPL v2 license. In particular: -- +- Code written by Trevor Perrin, Kees Bos, Sam Rushing, Dimitris Moraitis, Marcelo Fernandez, Martin von Loewis, Dave Baggett, Yngve Pettersen, and @@ -38,7 +38,7 @@ its author. See rijndael.py for details. Code written by Google is available under the following terms: -Copyright (c) 2008, The Chromium Authors +Copyright (c) 2008, The Chromium Authors All rights reserved. Redistribution and use in source and binary forms, with or without @@ -68,6 +68,29 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +Code written by Sidney Markowitz is available under the following terms: +Copyright (c) 2021 by Sidney Markowitz. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +- + Code written by Hubert Kario is available under the following terms: Copyright (c) 2014, Hubert Kario, Red Hat Inc. diff --git a/Makefile b/Makefile index 570bf7ec..6aa36b21 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Authors: +# Authors: # Trevor Perrin # Hubert Kario - test and test-dev # diff --git a/README.md b/README.md index 23a42557..4e836088 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Hubert Kario. TLS Lite was written (mostly) by Trevor Perrin. It includes code from Bram Cohen, Google, Kees Bos, Sam Rushing, Dimitris Moraitis, Marcelo Fernandez, Martin von Loewis, Dave Baggett, Yngve N. Pettersen (ported by Paul Sokolovsky), Mirko Dziadzka, David Benjamin, -and Hubert Kario. +Sidney Markowitz, and Hubert Kario. Original code in TLS Lite has either been dedicated to the public domain by its authors, or placed under a BSD-style license. See the LICENSE file for diff --git a/scripts/tls.py b/scripts/tls.py index a3f27ebe..79bb3c2c 100755 --- a/scripts/tls.py +++ b/scripts/tls.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Authors: +# Authors: # Trevor Perrin # Marcelo Fernandez - bugfix and NPN support # Martin von Loewis - python 3 port @@ -38,6 +38,7 @@ from tlslite.utils.dns_utils import is_valid_hostname from tlslite.utils.cryptomath import getRandomBytes from tlslite.constants import KeyUpdateMessageType +from tlslite.utils.compression import compression_algo_impls try: from tack.structures.Tack import Tack @@ -58,7 +59,7 @@ def printUsage(s=None): if tackpyLoaded: print(" tackpy : Loaded") else: - print(" tackpy : Not Loaded") + print(" tackpy : Not Loaded") if m2cryptoLoaded: print(" M2Crypto : Loaded") else: @@ -76,10 +77,30 @@ def printUsage(s=None): else: print(" GMPY2 : Not Loaded") + print("") + print("Compression algorithms:") + print(" zlib compress : Loaded") + print(" zlib decompress : Loaded") + print(" brotli compress : {0}".format( + "Loaded" if compression_algo_impls["brotli_compress"] + else "Not Loaded" + )) + print(" brotli decompress : {0}".format( + "Loaded" if compression_algo_impls["brotli_decompress"] + else "Not Loaded" + )) + print(" zstd decompress : {0}".format( + "Loaded" if compression_algo_impls["zstd_compress"] + else "Not Loaded" + )) + print(" zstd decompress : {0}".format( + "Loaded" if compression_algo_impls["zstd_decompress"] + else "Not Loaded" + )) print("") print("""Commands: - server + server [-c CERT] [-k KEY] [-t TACK] [-v VERIFIERDB] [-d DIR] [-l LABEL] [-L LENGTH] [--reqcert] [--param DHFILE] [--psk PSK] [--psk-ident IDENTITY] [--psk-sha384] [--ssl3] [--max-ver VER] [--tickets COUNT] [--cipherlist] @@ -144,8 +165,8 @@ def handleArgs(argv, argString, flagsList=[]): try: opts, argv = getopt.getopt(argv, getOptArgString, flagsList) except getopt.GetoptError as e: - printError(e) - # Default values if arg not present + printError(e) + # Default values if arg not present privateKey = None cert_chain = None virtual_hosts = [] @@ -367,6 +388,12 @@ def printGoodConnection(connection, seconds): print(" Extended Master Secret: {0}".format( connection.extendedMasterSecret)) print(" Session Resumed: {0}".format(connection.resumed)) + if connection.client_cert_compression_algo: + print(" Client compression algorithm used: {0}".format( + connection.client_cert_compression_algo)) + if connection.server_cert_compression_algo: + print(" Server compression algorithm used: {0}".format( + connection.server_cert_compression_algo)) def printExporter(connection, expLabel, expLength): if expLabel is None: @@ -378,7 +405,7 @@ def printExporter(connection, expLabel, expLength): print(" Exporter length: {0}".format(expLength)) print(" Keying material: {0}".format(exp)) - + def clientCmd(argv): (address, privateKey, cert_chain, virtual_hosts, username, password, expLabel, @@ -387,7 +414,7 @@ def clientCmd(argv): handleArgs(argv, "kcuplLa", ["psk=", "psk-ident=", "psk-sha384", "resumption", "ssl3", "max-ver=", "cipherlist="]) - + if (cert_chain and not privateKey) or (not cert_chain and privateKey): raise SyntaxError("Must specify CERT and KEY together") if (username and not password) or (not username and password): @@ -403,7 +430,7 @@ def clientCmd(argv): sock.connect(address) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) connection = TLSConnection(sock) - + settings = HandshakeSettings() if psk: settings.pskConfigs = [(psk_ident, psk, psk_hash)] @@ -418,13 +445,13 @@ def clientCmd(argv): try: start = time_stamp() if username and password: - connection.handshakeClientSRP(username, password, + connection.handshakeClientSRP(username, password, settings=settings, serverName=address[0]) else: connection.handshakeClientCert(cert_chain, privateKey, settings=settings, serverName=address[0], alpn=alpn) stop = time_stamp() - print("Handshake success") + print("Handshake success") except TLSLocalAlert as a: if a.description == AlertDescription.user_canceled: print(str(a)) @@ -544,7 +571,7 @@ def serverCmd(argv): print("Using Tacks...") if reqCert: print("Asking for client certificates...") - + ############# sessionCache = SessionCache() username = None diff --git a/tests/tlstest.py b/tests/tlstest.py index 18a64b73..9ce40f4d 100755 --- a/tests/tlstest.py +++ b/tests/tlstest.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Authors: +# Authors: # Trevor Perrin # Kees Bos - Added tests for XML-RPC # Dimitris Moraitis - Anon ciphersuites @@ -48,7 +48,7 @@ try: from tack.structures.Tack import Tack - + except ImportError: pass @@ -56,10 +56,10 @@ def printUsage(s=None): if m2cryptoLoaded: crypto = "M2Crypto/OpenSSL" else: - crypto = "Python crypto" + crypto = "Python crypto" if s: print("ERROR: %s" % s) - print("""\ntls.py version %s (using %s) + print("""\ntls.py version %s (using %s) Commands: server HOST:PORT DIRECTORY @@ -67,7 +67,7 @@ def printUsage(s=None): client HOST:PORT DIRECTORY """ % (__version__, crypto)) sys.exit(-1) - + def testConnClient(conn): b1 = os.urandom(1) @@ -92,9 +92,9 @@ def testConnClient(conn): assert r1000 == b1000 def clientTestCmd(argv): - + address = argv[0] - dir = argv[1] + dir = argv[1] #Split address into hostname/port tuple address = address.split(":") @@ -137,6 +137,8 @@ def connect(): assert(connection.session.cipherSuite in constants.CipherSuite.aeadSuites) assert(connection.encryptThenMAC == False) assert connection.session.appProto is None + assert connection.server_cert_compression_algo == "zlib" + assert connection.client_cert_compression_algo is None connection.close() test_no += 1 @@ -174,6 +176,21 @@ def connect(): test_no += 1 + print("Test {0} - good X.509 TLSv1.3 (no cert_comp)".format(test_no)) + synchro.recv(1) + settings = HandshakeSettings() + settings.certificate_compression_receive = [] + settings.certificate_compression_send = [] + connection = connect() + connection.handshakeClientCert(serverName=address[0], + settings=settings) + testConnClient(connection) + assert connection.server_cert_compression_algo is None + assert connection.client_cert_compression_algo is None + connection.close() + + test_no += 1 + print("Test {0} - good X.509/w RSA-PSS sig".format(test_no)) synchro.recv(1) connection = connect() @@ -235,7 +252,7 @@ def connect(): settings.minVersion = (3,0) settings.maxVersion = (3,0) connection.handshakeClientCert(settings=settings) - testConnClient(connection) + testConnClient(connection) assert(isinstance(connection.session.serverCertChain, X509CertChain)) connection.close() @@ -670,7 +687,7 @@ def connect(): settings.cipherNames = ["rc4"] settings.maxVersion = (3, 3) connection.handshakeClientCert(settings=settings) - testConnClient(connection) + testConnClient(connection) assert(isinstance(connection.session.serverCertChain, X509CertChain)) assert(connection.session.cipherSuite == constants.CipherSuite.TLS_RSA_WITH_RC4_128_MD5) assert(connection.encryptThenMAC == False) @@ -689,8 +706,8 @@ def connect(): connection = connect() connection.handshakeClientCert(settings=settings) assert(connection.session.tackExt.tacks[0].getTackId() == "5lcbe.eyweo.yxuan.rw6xd.jtoz7") - assert(connection.session.tackExt.activation_flags == 1) - testConnClient(connection) + assert(connection.session.tackExt.activation_flags == 1) + testConnClient(connection) connection.close() test_no += 1 @@ -856,7 +873,7 @@ def connect(): print("Test {0} - good SRP: with X.509 certificate, TLSv1.0".format(test_no)) settings = HandshakeSettings() settings.minVersion = (3,1) - settings.maxVersion = (3,1) + settings.maxVersion = (3,1) synchro.recv(1) connection = connect() connection.handshakeClientSRP("test", "password", settings=settings) @@ -908,6 +925,8 @@ def connect(): connection.handshakeClientCert(x509Chain, x509Key) testConnClient(connection) assert isinstance(connection.session.serverCertChain, X509CertChain) + assert connection.server_cert_compression_algo == "zlib" + assert connection.client_cert_compression_algo == "zlib" connection.close() test_no += 1 @@ -1215,7 +1234,7 @@ def connect(): connection = connect() settings = HandshakeSettings() settings.maxVersion = (3, 3) - connection.handshakeClientSRP("test", "garbage", serverName=address[0], + connection.handshakeClientSRP("test", "garbage", serverName=address[0], session=session, settings=settings) testConnClient(connection) #Don't close! -- see below @@ -1294,7 +1313,7 @@ def connect(): settings.cipherNames = [cipher] settings.cipherImplementations = [implementation, "python"] settings.minVersion = (3,1) - settings.maxVersion = (3,1) + settings.maxVersion = (3,1) connection.handshakeClientCert(settings=settings) testConnClient(connection) print("%s %s" % (connection.getCipherName(), connection.getCipherImplementation())) @@ -1875,7 +1894,7 @@ def serverTestCmd(argv): address = argv[0] dir = argv[1] - + #Split address into hostname/port tuple address = address.split(":") address = ( address[0], int(address[1]) ) @@ -2010,7 +2029,7 @@ def connect(): synchro.send(b'R') connection = connect() connection.handshakeServer(anon=True) - testConnServer(connection) + testConnServer(connection) connection.close() test_no += 1 @@ -2022,6 +2041,8 @@ def connect(): assert connection.session.serverName == address[0] assert connection.extendedMasterSecret assert connection.session.appProto is None + assert connection.server_cert_compression_algo == "zlib" + assert connection.client_cert_compression_algo is None testConnServer(connection) connection.close() @@ -2056,6 +2077,17 @@ def connect(): test_no += 1 + print("Test {0} - good X.509 TLSv1.3 (no cert_comp)".format(test_no)) + synchro.send(b'R') + connection = connect() + connection.handshakeServer(certChain=x509Chain, privateKey=x509Key) + assert connection.server_cert_compression_algo is None + assert connection.client_cert_compression_algo is None + testConnServer(connection) + connection.close() + + test_no += 1 + print("Test {0} - good X.509/w RSA-PSS sig".format(test_no)) synchro.send(b'R') connection = connect() @@ -2600,7 +2632,7 @@ def connect(): connection = connect() connection.handshakeServer(verifierDB=verifierDB, \ certChain=x509Chain, privateKey=x509Key) - testConnServer(connection) + testConnServer(connection) connection.close() test_no += 1 @@ -2632,6 +2664,8 @@ def connect(): connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, reqCert=True) testConnServer(connection) assert(isinstance(connection.session.clientCertChain, X509CertChain)) + assert connection.server_cert_compression_algo == "zlib" + assert connection.client_cert_compression_algo == "zlib" connection.close() test_no += 1 @@ -2905,7 +2939,7 @@ def connect(): sessionCache = SessionCache() connection = connect() connection.handshakeServer(verifierDB=verifierDB, sessionCache=sessionCache) - assert(connection.session.serverName == address[0]) + assert(connection.session.serverName == address[0]) testConnServer(connection) connection.close() @@ -2916,7 +2950,7 @@ def connect(): connection = connect() connection.handshakeServer(verifierDB=verifierDB, sessionCache=sessionCache) assert(connection.session.serverName == address[0]) - testConnServer(connection) + testConnServer(connection) #Don't close! -- see next test test_no += 1 @@ -3048,7 +3082,7 @@ def server_bind(self): synchro.send(b'R') connection = connect() settings = HandshakeSettings() - connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, + connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, settings=settings, nextProtos=[b"http/1.1"]) testConnServer(connection) connection.close() @@ -3059,7 +3093,7 @@ def server_bind(self): synchro.send(b'R') connection = connect() settings = HandshakeSettings() - connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, + connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, settings=settings, nextProtos=[b"spdy/2", b"http/1.1"]) testConnServer(connection) connection.close() @@ -3070,7 +3104,7 @@ def server_bind(self): synchro.send(b'R') connection = connect() settings = HandshakeSettings() - connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, + connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, settings=settings, nextProtos=[b"http/1.1", b"spdy/2"]) testConnServer(connection) connection.close() @@ -3081,7 +3115,7 @@ def server_bind(self): synchro.send(b'R') connection = connect() settings = HandshakeSettings() - connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, + connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, settings=settings, nextProtos=[b"spdy/2", b"http/1.1"]) testConnServer(connection) connection.close() @@ -3092,7 +3126,7 @@ def server_bind(self): synchro.send(b'R') connection = connect() settings = HandshakeSettings() - connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, + connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, settings=settings, nextProtos=[b"http/1.1", b"spdy/2", b"spdy/3"]) testConnServer(connection) connection.close() @@ -3103,7 +3137,7 @@ def server_bind(self): synchro.send(b'R') connection = connect() settings = HandshakeSettings() - connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, + connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, settings=settings, nextProtos=[b"spdy/3", b"spdy/2"]) testConnServer(connection) connection.close() @@ -3114,7 +3148,7 @@ def server_bind(self): synchro.send(b'R') connection = connect() settings = HandshakeSettings() - connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, + connection.handshakeServer(certChain=x509Chain, privateKey=x509Key, settings=settings, nextProtos=[]) testConnServer(connection) connection.close() diff --git a/tlslite/constants.py b/tlslite/constants.py index dd958c57..1ff3a86a 100644 --- a/tlslite/constants.py +++ b/tlslite/constants.py @@ -1,4 +1,4 @@ -# Authors: +# Authors: # Trevor Perrin # Google - defining ClientCertificateType # Google (adapted by Sam Rushing) - NPN support @@ -129,6 +129,7 @@ class HandshakeType(TLSEnum): finished = 20 certificate_status = 22 key_update = 24 # TLS 1.3 + compressed_certificate = 25 # TLS 1.3 next_protocol = 67 message_hash = 254 # TLS 1.3 @@ -168,6 +169,7 @@ class ExtensionType(TLSEnum): client_hello_padding = 21 # RFC 7685 encrypt_then_mac = 22 # RFC 7366 extended_master_secret = 23 # RFC 7627 + compress_certificate = 27 # RFC 8879 record_size_limit = 28 # RFC 8449 session_ticket = 35 # RFC 5077 extended_random = 40 # draft-rescorla-tls-extended-random-02 @@ -581,6 +583,17 @@ class PskKeyExchangeMode(TLSEnum): psk_dhe_ke = 1 +class CertificateCompressionAlgorithm(TLSEnum): + """ + Compression algorithms used for the compression of certificates + from RFC 8879. + """ + + zlib = 1 + brotli = 2 + zstd = 3 + + class CipherSuite: """ diff --git a/tlslite/extensions.py b/tlslite/extensions.py index 7d72ce1c..31a5f60d 100644 --- a/tlslite/extensions.py +++ b/tlslite/extensions.py @@ -12,7 +12,7 @@ from .constants import NameType, ExtensionType, CertificateStatusType, \ SignatureAlgorithm, HashAlgorithm, SignatureScheme, \ PskKeyExchangeMode, CertificateType, GroupName, ECPointFormat, \ - HeartbeatMode + HeartbeatMode, CertificateCompressionAlgorithm from .errors import TLSInternalError @@ -2158,43 +2158,54 @@ def __repr__(self): self.ticket) -TLSExtension._universalExtensions = \ - { - ExtensionType.server_name: SNIExtension, - ExtensionType.status_request: StatusRequestExtension, - ExtensionType.cert_type: ClientCertTypeExtension, - ExtensionType.supported_groups: SupportedGroupsExtension, - ExtensionType.ec_point_formats: ECPointFormatsExtension, - ExtensionType.srp: SRPExtension, - ExtensionType.signature_algorithms: SignatureAlgorithmsExtension, - ExtensionType.alpn: ALPNExtension, - ExtensionType.supports_npn: NPNExtension, - ExtensionType.client_hello_padding: PaddingExtension, - ExtensionType.renegotiation_info: RenegotiationInfoExtension, - ExtensionType.heartbeat: HeartbeatExtension, - ExtensionType.supported_versions: SupportedVersionsExtension, - ExtensionType.key_share: ClientKeyShareExtension, - ExtensionType.signature_algorithms_cert: - SignatureAlgorithmsCertExtension, - ExtensionType.pre_shared_key: PreSharedKeyExtension, - ExtensionType.psk_key_exchange_modes: PskKeyExchangeModesExtension, - ExtensionType.cookie: CookieExtension, - ExtensionType.record_size_limit: RecordSizeLimitExtension, - ExtensionType.session_ticket: SessionTicketExtension} - -TLSExtension._serverExtensions = \ - { - ExtensionType.cert_type: ServerCertTypeExtension, - ExtensionType.tack: TACKExtension, - ExtensionType.key_share: ServerKeyShareExtension, - ExtensionType.supported_versions: SrvSupportedVersionsExtension, - ExtensionType.pre_shared_key: SrvPreSharedKeyExtension} - -TLSExtension._certificateExtensions = \ - { - ExtensionType.status_request: CertificateStatusExtension} - -TLSExtension._hrrExtensions = \ - { - ExtensionType.key_share: HRRKeyShareExtension, - ExtensionType.supported_versions: SrvSupportedVersionsExtension} +class CompressedCertificateExtension(VarListExtension): + """Client and server compress certificate extension from RFC 8879""" + + def __init__(self): + """Create instance of class.""" + super(CompressedCertificateExtension, self).__init__( + 2, 1, 'algorithms', ExtensionType.compress_certificate, + CertificateCompressionAlgorithm) + + +TLSExtension._universalExtensions = { + ExtensionType.server_name: SNIExtension, + ExtensionType.status_request: StatusRequestExtension, + ExtensionType.cert_type: ClientCertTypeExtension, + ExtensionType.supported_groups: SupportedGroupsExtension, + ExtensionType.ec_point_formats: ECPointFormatsExtension, + ExtensionType.srp: SRPExtension, + ExtensionType.signature_algorithms: SignatureAlgorithmsExtension, + ExtensionType.alpn: ALPNExtension, + ExtensionType.supports_npn: NPNExtension, + ExtensionType.client_hello_padding: PaddingExtension, + ExtensionType.renegotiation_info: RenegotiationInfoExtension, + ExtensionType.heartbeat: HeartbeatExtension, + ExtensionType.supported_versions: SupportedVersionsExtension, + ExtensionType.key_share: ClientKeyShareExtension, + ExtensionType.signature_algorithms_cert: + SignatureAlgorithmsCertExtension, + ExtensionType.pre_shared_key: PreSharedKeyExtension, + ExtensionType.psk_key_exchange_modes: PskKeyExchangeModesExtension, + ExtensionType.cookie: CookieExtension, + ExtensionType.record_size_limit: RecordSizeLimitExtension, + ExtensionType.session_ticket: SessionTicketExtension, + ExtensionType.compress_certificate: CompressedCertificateExtension +} + +TLSExtension._serverExtensions = { + ExtensionType.cert_type: ServerCertTypeExtension, + ExtensionType.tack: TACKExtension, + ExtensionType.key_share: ServerKeyShareExtension, + ExtensionType.supported_versions: SrvSupportedVersionsExtension, + ExtensionType.pre_shared_key: SrvPreSharedKeyExtension +} + +TLSExtension._certificateExtensions = { + ExtensionType.status_request: CertificateStatusExtension +} + +TLSExtension._hrrExtensions = { + ExtensionType.key_share: HRRKeyShareExtension, + ExtensionType.supported_versions: SrvSupportedVersionsExtension +} diff --git a/tlslite/handshakesettings.py b/tlslite/handshakesettings.py index 38e560a2..fef9a40b 100644 --- a/tlslite/handshakesettings.py +++ b/tlslite/handshakesettings.py @@ -11,6 +11,7 @@ from .utils import cryptomath from .utils import cipherfactory from .utils.compat import ecdsaAllCurves, int_types +from .utils.compression import compression_algo_impls CIPHER_NAMES = ["chacha20-poly1305", "aes256gcm", "aes128gcm", @@ -62,6 +63,18 @@ "aes128ccm_8", "aes256ccm", "aes256ccm_8"] PSK_MODES = ["psk_dhe_ke", "psk_ke"] +ALL_COMPRESSION_ALGOS_SEND = ["zlib"] +if compression_algo_impls["brotli_compress"]: + ALL_COMPRESSION_ALGOS_SEND.append('brotli') +if compression_algo_impls["zstd_compress"]: + ALL_COMPRESSION_ALGOS_SEND.append('zstd') + +ALL_COMPRESSION_ALGOS_RECEIVE = ["zlib"] +if compression_algo_impls["brotli_decompress"]: + ALL_COMPRESSION_ALGOS_RECEIVE.append('brotli') +if compression_algo_impls["zstd_decompress"]: + ALL_COMPRESSION_ALGOS_RECEIVE.append('zstd') + class Keypair(object): """ @@ -353,6 +366,20 @@ class HandshakeSettings(object): :vartype keyExchangeNames: list :ivar keyExchangeNames: Enabled key exchange types for the connection, influences selected cipher suites. + + :vartype certificate_compression_send: list(str) + :ivar certificate_compression_send: a list of compression algorithms that + will be used to compress the certificate if compress_cerificate(27) + extension is supported in the handshake. This option is for when a + certificate was send/compressed by this peer. + + :vartype certificate_compression_receive: list(str) + :ivar certificate_compression_receive: a list of compression algorithms + that will be used to compress the certificate if + compress_cerificate(27) extension is supported in the handshake. This + option is for when a certificate was received/decompressed by this + peer. + """ def _init_key_settings(self): @@ -397,6 +424,11 @@ def _init_misc_extensions(self): self.ticket_count = 2 self.record_size_limit = 2**14 + 1 # TLS 1.3 includes content type + # Certificate compression + self.certificate_compression_send = list(ALL_COMPRESSION_ALGOS_SEND) + self.certificate_compression_receive = \ + list(ALL_COMPRESSION_ALGOS_RECEIVE) + def __init__(self): """Initialise default values for settings.""" self._init_key_settings() @@ -582,6 +614,8 @@ def _sanityCheckEMSExtension(other): @staticmethod def _sanityCheckExtensions(other): """Check if set extension settings are sane""" + not_matching = HandshakeSettings._not_matching + if other.useEncryptThenMAC not in (True, False): raise ValueError("useEncryptThenMAC can only be True or False") @@ -601,6 +635,30 @@ def _sanityCheckExtensions(other): HandshakeSettings._sanityCheckEMSExtension(other) + if other.certificate_compression_send: + if not hasattr(other.certificate_compression_send, '__iter__')\ + or isinstance(other.certificate_compression_send, str): + raise ValueError("certificate_compression must be an iterable " + "of strings") + + unknownAlgos = not_matching(other.certificate_compression_send, + ALL_COMPRESSION_ALGOS_SEND) + if unknownAlgos: + raise ValueError("Unknown compression algorithm: '{0}'" + .format(unknownAlgos)) + + if other.certificate_compression_receive: + if not hasattr(other.certificate_compression_receive, '__iter__')\ + or isinstance(other.certificate_compression_receive, str): + raise ValueError("certificate_compression must be an iterable " + "of strings") + + unknownAlgos = not_matching(other.certificate_compression_receive, + ALL_COMPRESSION_ALGOS_RECEIVE) + if unknownAlgos: + raise ValueError("Unknown compression algorithm: '{0}'" + .format(unknownAlgos)) + @staticmethod def _not_allowed_len(values, sieve): """Return True if length of any item in values is not in sieve.""" @@ -675,6 +733,9 @@ def _copy_extension_settings(self, other): other.max_early_data = self.max_early_data other.ticket_count = self.ticket_count other.record_size_limit = self.record_size_limit + other.certificate_compression_send = self.certificate_compression_send + other.certificate_compression_receive = \ + self.certificate_compression_receive @staticmethod def _remove_all_matches(values, needle): diff --git a/tlslite/messages.py b/tlslite/messages.py index 1354cd14..6183d1ea 100644 --- a/tlslite/messages.py +++ b/tlslite/messages.py @@ -21,6 +21,7 @@ from .utils.deprecations import deprecated_attrs, deprecated_params from .extensions import * from .utils.format_output import none_as_unknown +from .utils.compression import compression_algo_impls class RecordHeader(object): @@ -2449,3 +2450,131 @@ def write(self): writer = Writer() writer.add(self.message_type, 1) return self.postWrite(writer) + + +class CompressedCertificate(Certificate): + + def __init__(self, certificateType, version=(3, 2)): + super(CompressedCertificate, self).__init__(certificateType, version) + self.handshakeType = HandshakeType.compressed_certificate + self.compression_algo = None + self._compression_cache = None + self._msg_len_cache = None + + def _compress(self, msg): + if not ( + (self.compression_algo == CertificateCompressionAlgorithm.zlib) or + (self.compression_algo == CertificateCompressionAlgorithm.brotli + and compression_algo_impls["brotli_compress"]) or + (self.compression_algo == CertificateCompressionAlgorithm.zstd + and compression_algo_impls["zstd_compress"]) + ): + raise ValueError("Unknown compression algorithm code: {0}" + .format(self.compression_algo)) + + if not isinstance(msg, bytes): + msg = bytes(msg) + + if self.compression_algo == CertificateCompressionAlgorithm.zlib: + compressed_msg = zlib.compress(msg) + elif self.compression_algo == CertificateCompressionAlgorithm.brotli: + compressed_msg = compression_algo_impls["brotli_compress"](msg) + else: + assert self.compression_algo == \ + CertificateCompressionAlgorithm.zstd + compressed_msg = compression_algo_impls["zstd_compress"](msg) + + return compressed_msg + + def _decompress(self, compressed_msg, expected_length): + if not ( + (self.compression_algo == CertificateCompressionAlgorithm.zlib) or + (self.compression_algo == CertificateCompressionAlgorithm.brotli + and compression_algo_impls["brotli_decompress"]) or + (self.compression_algo == CertificateCompressionAlgorithm.zstd + and compression_algo_impls["zstd_decompress"]) + ): + raise BadCertificateError("Unknown compression algorithm code: {0}" + .format(self.compression_algo)) + + if not isinstance(compressed_msg, bytes): + compressed_msg = bytes(compressed_msg) + + try: + if self.compression_algo == CertificateCompressionAlgorithm.zlib: + decompressed_msg = zlib.decompress( + compressed_msg, 15, expected_length) + elif self.compression_algo == \ + CertificateCompressionAlgorithm.brotli: + if compression_algo_impls["brotli_accepts_limit"]: + decompressed_msg = \ + compression_algo_impls["brotli_decompress"]( + compressed_msg, expected_length) + else: + decompressed_msg = \ + compression_algo_impls["brotli_decompress"]( + compressed_msg) + else: + assert self.compression_algo == \ + CertificateCompressionAlgorithm.zstd + if compression_algo_impls["zstd_accepts_limit"]: + decompressed_msg = \ + compression_algo_impls["zstd_decompress"]( + compressed_msg, expected_length) + else: + decompressed_msg = \ + compression_algo_impls["zstd_decompress"]( + compressed_msg) + except Exception: + raise BadCertificateError("Error on decompressing the message.") + + if len(decompressed_msg) != expected_length: + raise BadCertificateError( + "Decompressed message doesn't much length.") + + return decompressed_msg + + def create(self, compression_algo, cert_chain, context=b''): + """Create CompressedCertificate message.""" + super(CompressedCertificate, self).create(cert_chain, context) + self.compression_algo = compression_algo + self._compression_cache = None + self._msg_len_cache = None + return self + + def parse(self, p): + """Deserialize CompressedCertificate message from parser.""" + p.startLengthCheck(3) + self.compression_algo = p.get(2) + expected_length = p.get(3) + compressed_msg = p.getVarBytes(3) + p.stopLengthCheck() + certificate_msg = self._decompress(compressed_msg, expected_length) + + writer = Writer() + writer.add(expected_length, 3) + writer.bytes += certificate_msg + parser = Parser(writer.bytes) + super(CompressedCertificate, self).parse(parser) + + return self + + def write(self): + """Serialise CompressedCertificate message.""" + if not self._compression_cache: + certificate_msg = super(CompressedCertificate, self).write() + certificate_msg = certificate_msg[4:] + self._msg_len_cache = len(certificate_msg) + self._compression_cache = self._compress(certificate_msg) + + writer = Writer() + writer.add(self.compression_algo, 2) + writer.add(self._msg_len_cache, 3) + writer.add(len(self._compression_cache), 3) + writer.bytes += self._compression_cache + return self.postWrite(writer) + + def __repr__(self): + return "Compressed {0}".format( + super(CompressedCertificate, self).__repr__() + ) diff --git a/tlslite/tlsconnection.py b/tlslite/tlsconnection.py index 582097a7..06cf2726 100644 --- a/tlslite/tlsconnection.py +++ b/tlslite/tlsconnection.py @@ -39,6 +39,7 @@ from .handshakehelpers import HandshakeHelpers from .utils.cipherfactory import createAESCCM, createAESCCM_8, \ createAESGCM, createCHACHA20 +from .utils.compression import choose_compression_send_algo class TLSConnection(TLSRecordLayer): """ @@ -61,6 +62,18 @@ class TLSConnection(TLSRecordLayer): framework like asyncore or Twisted which TLS Lite integrates with (see :py:class:`~.integration.tlsasyncdispatchermixin.TLSAsyncDispatcherMixIn`). + + :vartype client_cert_compression_algo: string + :ivar client_cert_compression_algo: Set to the compression algorithm used + for the compression of the server certificate. In the case of multiple + post-handshake authentication only the algorithm of the last + certificate compression is reflected. If certificate compression wasn't + used then it is set to None. + + :vartype server_cert_compression_algo: string + :ivar server_cert_compression_algo: Set to the compression algorithm used + for the compression of the server certificate. If certificate + compression wasn't used then it is set to None. """ def __init__(self, sock): @@ -86,6 +99,8 @@ def __init__(self, sock): # used only for TLS 1.2 and earlier self._peer_record_size_limit = None self._pha_supported = False + self.client_cert_compression_algo = None + self.server_cert_compression_algo = None def keyingMaterialExporter(self, label, length=20): """Return keying material as described in RFC 5705 @@ -808,6 +823,17 @@ def _clientSendClientHello(self, settings, session, srpUsername, extensions.append(SessionTicketExtension().create( bytearray(0))) + # when TLS 1.3 advertised, send also compress_certificate extension + if ( + next((i for i in settings.versions if i >= (3, 4)), None) + and settings.certificate_compression_receive + ): + algos_numbers = [getattr(CertificateCompressionAlgorithm, algo) + for algo + in settings.certificate_compression_receive] + extensions.append(CompressedCertificateExtension().create( + algos_numbers)) + # don't send empty list of extensions or extensions in SSLv3 if not extensions or settings.maxVersion == (3, 0): extensions = None @@ -825,7 +851,7 @@ def _clientSendClientHello(self, settings, session, srpUsername, clientHello = ClientHello() clientHello.create(sent_version, getRandomBytes(32), session.sessionID, wireCipherSuites, - certificateTypes, + certificateTypes, session.srpUsername, reqTack, nextProtos is not None, session.serverName, @@ -836,9 +862,9 @@ def _clientSendClientHello(self, settings, session, srpUsername, clientHello = ClientHello() clientHello.create(sent_version, getRandomBytes(32), session_id, wireCipherSuites, - certificateTypes, + certificateTypes, srpUsername, - reqTack, nextProtos is not None, + reqTack, nextProtos is not None, serverName, extensions=extensions) @@ -1083,7 +1109,7 @@ def _clientGetServerHello(self, settings, session, clientHello): AlertDescription.illegal_parameter, "Server responded with incorrect compression method"): yield result - if serverHello.tackExt: + if serverHello.tackExt: if not clientHello.tack: for result in self._sendError(\ AlertDescription.illegal_parameter, @@ -1297,10 +1323,20 @@ def _clientTLS13Handshake(self, settings, session, clientHello, # if we negotiated PSK then Certificate is not sent certificate_request = None certificate = None + + client_hello_comp_cert_ext = clientHello.getExtension( + ExtensionType.compress_certificate) + + if client_hello_comp_cert_ext: + expected_msg = (HandshakeType.certificate_request, + HandshakeType.certificate, + HandshakeType.compressed_certificate) + else: + expected_msg = (HandshakeType.certificate_request, + HandshakeType.certificate) + if not sr_psk: - for result in self._getMsg(ContentType.handshake, - (HandshakeType.certificate_request, - HandshakeType.certificate), + for result in self._getMsg(ContentType.handshake, expected_msg, CertificateType.x509): if result in (0, 1): yield result @@ -1310,15 +1346,26 @@ def _clientTLS13Handshake(self, settings, session, clientHello, if isinstance(result, CertificateRequest): certificate_request = result - # we got CertificateRequest so now we'll get Certificate - for result in self._getMsg(ContentType.handshake, - HandshakeType.certificate, + if client_hello_comp_cert_ext: + expected_msg = (HandshakeType.certificate, + HandshakeType.compressed_certificate) + else: + expected_msg = (HandshakeType.certificate) + + # we got CertificateRequest so now we'll get Certificate or + # Compressed Certificate + for result in self._getMsg(ContentType.handshake, expected_msg, CertificateType.x509): if result in (0, 1): yield result else: break + if isinstance(result, CompressedCertificate): + self.server_cert_compression_algo = \ + CertificateCompressionAlgorithm.toStr( + result.compression_algo) + certificate = result assert isinstance(certificate, Certificate) @@ -1418,8 +1465,6 @@ def _clientTLS13Handshake(self, settings, session, clientHello, server_finish_hs, prfName) if certificate_request: - client_certificate = Certificate(serverHello.certificate_type, - self.version) if clientCertChain: # Check to make sure we have the same type of certificates the # server requested @@ -1430,7 +1475,11 @@ def _clientTLS13Handshake(self, settings, session, clientHello, "Client certificate is of wrong type"): yield result - client_certificate.create(clientCertChain) + client_certificate = self._create_cert_msg( + "client", clientHello, settings.certificate_compression_send, + clientCertChain, serverHello.certificate_type, + version=self.version) + # we need to send the message even if we don't have a certificate for result in self._sendMsg(client_certificate): yield result @@ -1605,7 +1654,7 @@ def _clientSelectNextProto(self, nextProtos, serverHello): # # !!! We assume the client may have specified nextProtos as a list of # strings so we convert them to bytearrays (it's awkward to require - # the user to specify a list of bytearrays or "bytes", and in + # the user to specify a list of bytearrays or "bytes", and in # Python 2.6 bytes() is just an alias for str() anyways... if nextProtos is not None and serverHello.next_protos is not None: for p in nextProtos: @@ -1617,7 +1666,7 @@ def _clientSelectNextProto(self, nextProtos, serverHello): # the client SHOULD select the first protocol it supports. return bytearray(nextProtos[0]) return None - + def _clientResume(self, session, serverHello, clientRandom, nextProto, settings): @@ -1778,7 +1827,6 @@ def _clientKeyExchange(self, settings, cipherSuite, "Server doesn't accept any sigalgs we support: " + str(certificateRequest.supported_signature_algs)): yield result - clientCertificate = Certificate(certificateType) if clientCertChain: #Check to make sure we have the same type of @@ -1790,7 +1838,10 @@ def _clientKeyExchange(self, settings, cipherSuite, "Client certificate is of wrong type"): yield result - clientCertificate.create(clientCertChain) + clientCertificate = self._create_cert_msg( + "client", certificateRequest, + settings.certificate_compression_send, clientCertChain, + certificateType) # we need to send the message even if we don't have a certificate for result in self._sendMsg(clientCertificate): yield result @@ -1855,8 +1906,8 @@ def _clientFinished(self, premasterSecret, clientRandom, serverRandom, cipherSuite, clientRandom, serverRandom) - self._calcPendingStates(cipherSuite, masterSecret, - clientRandom, serverRandom, + self._calcPendingStates(cipherSuite, masterSecret, + clientRandom, serverRandom, cipherImplementations) #Exchange ChangeCipherSpec and Finished messages @@ -1967,13 +2018,13 @@ def _clientGetKeyFromChain(self, certificate, settings, tack_ext=None): if tackpyLoaded: if not tack_ext: tack_ext = cert_chain.getTackExt() - + # If there's a TACK (whether via TLS or TACK Cert), check that it - # matches the cert chain + # matches the cert chain if tack_ext and tack_ext.tacks: for tack in tack_ext.tacks: if not cert_chain.checkTack(tack): - for result in self._sendError( + for result in self._sendError( AlertDescription.illegal_parameter, "Other party's TACK doesn't match their public key"): yield result @@ -1989,7 +2040,7 @@ def _clientGetKeyFromChain(self, certificate, settings, tack_ext=None): def handshakeServer(self, verifierDB=None, certChain=None, privateKey=None, reqCert=False, sessionCache=None, settings=None, checker=None, - reqCAs = None, + reqCAs = None, tacks=None, activationFlags=0, nextProtos=None, anon=False, alpn=None, sni=None): """Perform a handshake in the role of server. @@ -2090,7 +2141,7 @@ def handshakeServer(self, verifierDB=None, def handshakeServerAsync(self, verifierDB=None, certChain=None, privateKey=None, reqCert=False, sessionCache=None, settings=None, checker=None, - reqCAs=None, + reqCAs=None, tacks=None, activationFlags=0, nextProtos=None, anon=False, alpn=None, sni=None ): @@ -2108,9 +2159,9 @@ def handshakeServerAsync(self, verifierDB=None, handshaker = self._handshakeServerAsyncHelper(\ verifierDB=verifierDB, cert_chain=certChain, privateKey=privateKey, reqCert=reqCert, - sessionCache=sessionCache, settings=settings, - reqCAs=reqCAs, - tacks=tacks, activationFlags=activationFlags, + sessionCache=sessionCache, settings=settings, + reqCAs=reqCAs, + tacks=tacks, activationFlags=activationFlags, nextProtos=nextProtos, anon=anon, alpn=alpn, sni=sni) for result in self._handshakeWrapperAsync(handshaker, checker): yield result @@ -2118,8 +2169,8 @@ def handshakeServerAsync(self, verifierDB=None, def _handshakeServerAsyncHelper(self, verifierDB, cert_chain, privateKey, reqCert, sessionCache, - settings, reqCAs, - tacks, activationFlags, + settings, reqCAs, + tacks, activationFlags, nextProtos, anon, alpn, sni): self._handshakeStart(client=False) @@ -2136,7 +2187,7 @@ def _handshakeServerAsyncHelper(self, verifierDB, if privateKey and not cert_chain: raise ValueError("Caller passed a privateKey but no cert_chain") if reqCAs and not reqCert: - raise ValueError("Caller passed reqCAs but not reqCert") + raise ValueError("Caller passed reqCAs but not reqCert") if cert_chain and not isinstance(cert_chain, X509CertChain): raise ValueError("Unrecognized certificate type") if activationFlags and not tacks: @@ -2153,7 +2204,7 @@ def _handshakeServerAsyncHelper(self, verifierDB, # OK Start exchanging messages # ****************************** - + # Handle ClientHello and resumption for result in self._serverGetClientHello(settings, privateKey, cert_chain, @@ -2161,8 +2212,8 @@ def _handshakeServerAsyncHelper(self, verifierDB, anon, alpn, sni): if result in (0,1): yield result elif result == None: - self._handshakeDone(resumed=True) - return # Handshake was resumed, we're done + self._handshakeDone(resumed=True) + return # Handshake was resumed, we're done else: break (clientHello, version, cipherSuite, sig_scheme, privateKey, cert_chain) = result @@ -2191,7 +2242,7 @@ def _handshakeServerAsyncHelper(self, verifierDB, sessionID = getRandomBytes(32) else: sessionID = bytearray(0) - + if not clientHello.supports_npn: nextProtos = None @@ -2470,7 +2521,21 @@ def request_post_handshake_auth(self, settings=None): context = bytes(getRandomBytes(32)) certificate_request = CertificateRequest(self.version) - certificate_request.create(context=context, sig_algs=valid_sig_algs) + + extensions = [] + if self.version >= (3, 4): + if settings: + algos_numbers = [ + getattr(CertificateCompressionAlgorithm, algo) for algo + in settings.certificate_compression_receive + ] + else: + algos_numbers = [] + extensions.append(CompressedCertificateExtension().create( + algos_numbers)) + + certificate_request.create(context=context, sig_algs=valid_sig_algs, + extensions=extensions) self._cert_requests[context] = certificate_request @@ -2631,6 +2696,7 @@ def _serverTLS13Handshake(self, settings, clientHello, cipherSuite, srv_alpns, reqCert): """Perform a TLS 1.3 handshake""" prf_name, prf_size = self._getPRFParams(cipherSuite) + cert_req_comp_cert_ext = None secret = bytearray(prf_size) @@ -2831,12 +2897,27 @@ def _serverTLS13Handshake(self, settings, clientHello, cipherSuite, valid_sig_algs = self._sigHashesToList(cr_settings) assert valid_sig_algs + extensions = [] + if self.version >= (3, 4): + algos_numbers = [ + getattr(CertificateCompressionAlgorithm, algo) for algo + in settings.certificate_compression_receive + ] + cert_req_comp_cert_ext = CompressedCertificateExtension()\ + .create(algos_numbers) + extensions.append(cert_req_comp_cert_ext) + certificate_request = CertificateRequest(self.version) - certificate_request.create(context=ctx, sig_algs=valid_sig_algs) + certificate_request.create( + context=ctx, sig_algs=valid_sig_algs, + extensions=extensions) self._queue_message(certificate_request) - certificate = Certificate(CertificateType.x509, self.version) - certificate.create(serverCertChain, bytearray()) + certificate = self._create_cert_msg( + "server", clientHello, settings.certificate_compression_send, + serverCertChain, CertificateType.x509, bytearray(), + self.version) + self._queue_message(certificate) certificate_verify = CertificateVerify(self.version) @@ -2921,15 +3002,25 @@ def _serverTLS13Handshake(self, settings, clientHello, cipherSuite, client_cert_chain = None #Get [Certificate,] (if was requested) if reqCert and selected_psk is None: - for result in self._getMsg(ContentType.handshake, - HandshakeType.certificate, + if cert_req_comp_cert_ext: + expected_msg = (HandshakeType.certificate, + HandshakeType.compressed_certificate) + else: + expected_msg = (HandshakeType.certificate) + + for result in self._getMsg(ContentType.handshake, expected_msg, CertificateType.x509): if result in (0, 1): yield result else: break client_certificate = result - assert isinstance(client_certificate, Certificate) + if isinstance(client_certificate, CompressedCertificate): + self.client_cert_compression_algo = \ + CertificateCompressionAlgorithm.toStr( + client_certificate.compression_algo) + else: + assert isinstance(client_certificate, Certificate) client_cert_chain = client_certificate.cert_chain #Get and check CertificateVerify, if relevant @@ -3695,9 +3786,9 @@ def _serverGetClientHello(self, settings, private_key, cert_chain, yield result #Calculate pending connection states - self._calcPendingStates(session.cipherSuite, + self._calcPendingStates(session.cipherSuite, session.masterSecret, - clientHello.random, + clientHello.random, serverHello.random, settings.cipherImplementations) @@ -3986,13 +4077,14 @@ def _serverSRPKeyExchange(self, clientHello, serverHello, verifierDB, AlertDescription.insufficient_security): yield result - #Send ServerHello[, Certificate], ServerKeyExchange, - #ServerHelloDone + #Send ServerHello[, Certificate or Compressed Certificate], + #ServerKeyExchange, ServerHelloDone msgs = [] msgs.append(serverHello) if cipherSuite in CipherSuite.srpCertSuites: - certificateMsg = Certificate(CertificateType.x509) - certificateMsg.create(serverCertChain) + certificateMsg = self._create_cert_msg( + "server", clientHello, settings.certificate_compression_send, + serverCertChain, CertificateType.x509) msgs.append(certificateMsg) msgs.append(serverKeyExchange) msgs.append(ServerHelloDone()) @@ -4177,15 +4269,34 @@ def _serverCertKeyExchange(self, clientHello, serverHello, sigHashAlg, serverCertChain, keyExchange, reqCert, reqCAs, cipherSuite, settings): - #Send ServerHello, Certificate[, ServerKeyExchange] - #[, CertificateRequest], ServerHelloDone + #Send ServerHello, Certificate or Compressed Certificate + #[, ServerKeyExchange] [, CertificateRequest], ServerHelloDone msgs = [] # If we verify a client cert chain, return it clientCertChain = None msgs.append(serverHello) - msgs.append(Certificate(CertificateType.x509).create(serverCertChain)) + + client_hello_comp_cert_ext = clientHello.getExtension( + ExtensionType.compress_certificate) + chosen_compression_algo = choose_compression_send_algo( + self.version, client_hello_comp_cert_ext, + settings.certificate_compression_send) + + if chosen_compression_algo: + self.server_cert_compression_algo = \ + CertificateCompressionAlgorithm.toStr( + chosen_compression_algo) + certificate = CompressedCertificate(CertificateType.x509, + self.version) + certificate.create(chosen_compression_algo, serverCertChain, + bytearray()) + else: + certificate = Certificate(CertificateType.x509, self.version) + certificate.create(serverCertChain, bytearray()) + + msgs.append(certificate) try: serverKeyExchange = keyExchange.makeServerKeyExchange(sigHashAlg) except TLSInternalError as alert: @@ -4215,9 +4326,19 @@ def _serverCertKeyExchange(self, clientHello, serverHello, sigHashAlg, if cr_settings.dsaSigHashes: cert_types.append(ClientCertificateType.dss_sign) + extensions = [] + if self.version >= (3, 4): + algos_numbers = [ + getattr(CertificateCompressionAlgorithm, algo) for algo + in cr_settings.certificate_compression_receive + ] + extensions.append(CompressedCertificateExtension().create( + algos_numbers)) + certificateRequest.create(cert_types, reqCAs, - valid_sig_algs) + valid_sig_algs, + extensions=extensions) msgs.append(certificateRequest) msgs.append(ServerHelloDone()) for result in self._sendMsgs(msgs): @@ -4422,7 +4543,7 @@ def _serverFinished(self, premasterSecret, clientRandom, serverRandom, self.session.masterSecret = masterSecret #Calculate pending connection states - self._calcPendingStates(cipherSuite, masterSecret, + self._calcPendingStates(cipherSuite, masterSecret, clientRandom, serverRandom, cipherImplementations) @@ -4534,7 +4655,7 @@ def _getFinished(self, masterSecret, cipherSuite=None, #Switch to pending read state self._changeReadState() - #Server Finish - Are we waiting for a next protocol echo? + #Server Finish - Are we waiting for a next protocol echo? if expect_next_protocol: for result in self._getMsg(ContentType.handshake, HandshakeType.next_protocol): if result in (0,1): diff --git a/tlslite/tlsrecordlayer.py b/tlslite/tlsrecordlayer.py index 0cd31f28..805f6cf0 100644 --- a/tlslite/tlsrecordlayer.py +++ b/tlslite/tlsrecordlayer.py @@ -1,4 +1,4 @@ -# Authors: +# Authors: # Trevor Perrin # Google (adapted by Sam Rushing) - NPN support # Google - minimal padding @@ -19,6 +19,8 @@ from .utils.cryptomath import * from .utils.codec import Parser, BadCertificateError from .utils.lists import to_str_delimiter, getFirstMatching +from .utils.compression import compression_algo_impls, \ + choose_compression_send_algo from .errors import * from .messages import * from .mathtls import * @@ -333,9 +335,24 @@ def readAsync(self, max=None, min=1): HandshakeType.key_update, HandshakeType.certificate_request) elif self._cert_requests: - allowedHsTypes = (HandshakeType.new_session_ticket, - HandshakeType.key_update, - HandshakeType.certificate) + cert_req_with_comp_cert_ext = False + for _, cert_request in self._cert_requests.items(): + cert_req_comp_cert_ext = cert_request.getExtension( + ExtensionType.compress_certificate) + cert_req_with_comp_cert_ext = cert_req_with_comp_cert_ext \ + or cert_req_comp_cert_ext is not None + if cert_req_with_comp_cert_ext: + break + + if cert_req_with_comp_cert_ext: + allowedHsTypes = (HandshakeType.new_session_ticket, + HandshakeType.key_update, + HandshakeType.certificate, + HandshakeType.compressed_certificate) + else: + allowedHsTypes = (HandshakeType.new_session_ticket, + HandshakeType.key_update, + HandshakeType.certificate) constructor_type = CertificateType.x509 else: allowedHsTypes = (HandshakeType.new_session_ticket, @@ -367,6 +384,11 @@ def readAsync(self, max=None, min=1): # KeyUpdate messages are not solicited, while call with # min==0 are done to perform PHA try_once = True + elif isinstance(result, CompressedCertificate): + self.client_cert_compression_algo = \ + result.compression_algo + for result in self._handle_srv_pha(result): + yield result elif isinstance(result, Certificate): for result in self._handle_srv_pha(result): yield result @@ -502,7 +524,7 @@ def _decrefAsync(self): yield result alert = None # By default close the socket, since it's been observed - # that some other libraries will not respond to the + # that some other libraries will not respond to the # close_notify alert, thus leaving us hanging if we're # expecting it if self.closeSocket: @@ -613,7 +635,7 @@ def makefile(self, mode='r', bufsize=-1): # class, so that when fileobject.close() gets called, it will # close() us, causing the refcount to be decremented (decrefAsync). # - # If this is the last close() on the outstanding fileobjects / + # If this is the last close() on the outstanding fileobjects / # TLSConnection, then the "actual" close alerts will be sent, # socket closed, etc. @@ -656,11 +678,11 @@ def setsockopt(self, level, optname, value): def shutdown(self, how): """Shutdown the underlying socket.""" return self.sock.shutdown(how) - + def fileno(self): """Not implement in TLS Lite.""" raise NotImplementedError() - + #********************************************************* # Public Functions END @@ -679,8 +701,19 @@ def _handle_pha(self, cert_request): prf_size = 48 msgs = [] - msgs.append(Certificate(CertificateType.x509, self.version) - .create(cert, cert_request.certificate_request_context)) + + valid_compression_algos = ["zlib"] + if compression_algo_impls["brotli_compress"]: + valid_compression_algos.append("brotli") + if compression_algo_impls["zstd_compress"]: + valid_compression_algos.append("zstd") + + client_certificate = self._create_cert_msg( + 'client', cert_request, valid_compression_algos, cert, + CertificateType.x509, cert_request.certificate_request_context, + self.version) + + msgs.append(client_certificate) handshake_context.update(msgs[0].write()) if cert.x509List and p_key: # sign the CertificateVerify only when we have a private key to do @@ -1225,6 +1258,9 @@ def _getMsg(self, expectedType, secondaryType=None, constructorType=None): yield ServerHello().parse(p) elif subType == HandshakeType.certificate: yield Certificate(constructorType, self.version).parse(p) + elif subType == HandshakeType.compressed_certificate: + yield CompressedCertificate( + constructorType, self.version).parse(p) elif subType == HandshakeType.certificate_request: yield CertificateRequest(self.version).parse(p) elif subType == HandshakeType.certificate_verify: @@ -1482,3 +1518,36 @@ def send_keyupdate_request(self, message_type): self.session.cipherSuite, self.session.cl_app_secret, self.session.sr_app_secret) + + def _create_cert_msg(self, peer, request_msg, valid_compression_algos, + cert_chain, cert_type, cert_context=b'', + version=(3, 2)): + """ + Creates either a Certificate or a CompressedCertificate message + depending if the compress_certificate extension is present. + """ + + cert_req_comp_cert_ext = request_msg.getExtension( + ExtensionType.compress_certificate) + chosen_compression_algo = choose_compression_send_algo( + version, cert_req_comp_cert_ext, + valid_compression_algos) + + if chosen_compression_algo: + if peer == "server": + self.server_cert_compression_algo = \ + CertificateCompressionAlgorithm.toStr( + chosen_compression_algo) + else: + self.client_cert_compression_algo = \ + CertificateCompressionAlgorithm.toStr( + chosen_compression_algo) + + certificate_msg = CompressedCertificate(cert_type, version) + certificate_msg.create( + chosen_compression_algo, cert_chain, cert_context) + else: + certificate_msg = Certificate(cert_type, version) + certificate_msg.create(cert_chain, cert_context) + + return certificate_msg diff --git a/tlslite/utils/brotlidecpy/LICENCE b/tlslite/utils/brotlidecpy/LICENCE new file mode 100644 index 00000000..088e3d01 --- /dev/null +++ b/tlslite/utils/brotlidecpy/LICENCE @@ -0,0 +1,19 @@ +Copyright (c) 2021 by Sidney Markowitz. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/tlslite/utils/brotlidecpy/__init__.py b/tlslite/utils/brotlidecpy/__init__.py new file mode 100644 index 00000000..e0696cbe --- /dev/null +++ b/tlslite/utils/brotlidecpy/__init__.py @@ -0,0 +1,11 @@ +''' +This module it pure python brotli decompress. +Copied from https://github.com/sidney/brotlidecpy +''' + +from __future__ import absolute_import + +__version__ = "1.0.3" + +# noinspection PyUnresolvedReferences +from .decode import brotli_decompress_buffer as decompress diff --git a/tlslite/utils/brotlidecpy/bit_reader.py b/tlslite/utils/brotlidecpy/bit_reader.py new file mode 100644 index 00000000..42553a28 --- /dev/null +++ b/tlslite/utils/brotlidecpy/bit_reader.py @@ -0,0 +1,84 @@ +# Copyright 2021 Sidney Markowitz All Rights Reserved. +# Distributed under MIT license. +# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT + + +class BrotliBitReader: + """ + Wrap a bytes buffer to enable reading 0 < n <=24 bits at a time, or + transfer of arbitrary number of bytes + """ + + kBitMask = [ + 0x000000, 0x000001, 0x000003, 0x000007, 0x00000f, 0x00001f, 0x00003f, + 0x00007f, 0x0000ff, 0x0001ff, 0x0003ff, 0x0007ff, 0x000fff, 0x001fff, + 0x003fff, 0x007fff, 0x00ffff, 0x01ffff, 0x03ffff, 0x07ffff, 0x0fffff, + 0x1fffff, 0x3fffff, 0x7fffff, 0xffffff + ] + + def __init__(self, input_buffer): + self.buf_ = bytearray(input_buffer) + self.buf_len_ = len(input_buffer) + self.pos_ = 0 # byte position in stream + # current bit-reading position in current byte (number bits already + # read from byte, 0-7) + self.bit_pos_ = 0 + + def reset(self): + """Reset an initialized BrotliBitReader to start of input buffer""" + self.pos_ = 0 + self.bit_pos_ = 0 + + def read_bits(self, n_bits, bits_to_skip=None): + """ + Get n_bits unsigned integer treating input as little-endian byte + stream, maybe advancing input buffer pointer + + n_bits: is number of bits to read from input buffer. Set to None or 0 + to seek ahead ignoring the value + bits_to_skip: number of bits to advance in input_buffer, defaults to + n_bits if it is None pass in 0 to peek at the next n_bits of value + without advancing + + It is ok to have n_bits and bits_to_skip be different non-zero values + if that is what is wanted + + Returns: the next n_bits from the buffer as a little-endian integer, + 0 if n_bits is None or 0 + """ + val = 0 + if bits_to_skip is None: + bits_to_skip = n_bits + if n_bits: + bytes_shift = 0 + buf_pos = self.pos_ + bit_pos_when_done = n_bits + self.bit_pos_ + while bytes_shift < bit_pos_when_done: + if buf_pos >= self.buf_len_: + # if hit end of buffer, this simulates zero padding after + # end, which is correct + break + val |= self.buf_[buf_pos] << bytes_shift + bytes_shift += 8 + buf_pos += 1 + val = (val >> self.bit_pos_) & self.kBitMask[n_bits] + if bits_to_skip: + next_in_bits = self.bit_pos_ + bits_to_skip + self.bit_pos_ = next_in_bits & 7 + self.pos_ += next_in_bits >> 3 + return val + + def copy_bytes(self, dest_buffer, dest_pos, n_bytes): + """ + Copy bytes from input buffer. This will first skip to next byte + boundary if not already on one + """ + if self.bit_pos_ != 0: + self.bit_pos_ = 0 + self.pos_ += 1 + # call with n_bytes == 0 to just skip to next byte boundary + if n_bytes > 0: + new_pos = self.pos_ + n_bytes + memoryview(dest_buffer)[dest_pos:dest_pos+n_bytes] = \ + self.buf_[self.pos_:new_pos] + self.pos_ = new_pos diff --git a/tlslite/utils/brotlidecpy/brotli-dict b/tlslite/utils/brotlidecpy/brotli-dict new file mode 100644 index 00000000..a585c0e2 --- /dev/null +++ b/tlslite/utils/brotlidecpy/brotli-dict @@ -0,0 +1,432 @@ +timedownlifeleftbackcodedatashowonlysitecityopenjustlikefreeworktextyearoverbodyloveformbookplaylivelinehelphomesidemorewordlongthemviewfindpagedaysfullheadtermeachareafromtruemarkableuponhighdatelandnewsevennextcasebothpostusedmadehandherewhatnameLinkblogsizebaseheldmakemainuser') +holdendswithNewsreadweresigntakehavegameseencallpathwellplusmenufilmpartjointhislistgoodneedwayswestjobsmindalsologorichuseslastteamarmyfoodkingwilleastwardbestfirePageknowaway.pngmovethanloadgiveselfnotemuchfeedmanyrockicononcelookhidediedHomerulehostajaxinfoclublawslesshalfsomesuchzone100%onescareTimeracebluefourweekfacehopegavehardlostwhenparkkeptpassshiproomHTMLplanTypedonesavekeepflaglinksoldfivetookratetownjumpthusdarkcardfilefearstaykillthatfallautoever.comtalkshopvotedeepmoderestturnbornbandfellroseurl(skinrolecomeactsagesmeetgold.jpgitemvaryfeltthensenddropViewcopy1.0"stopelseliestourpack.gifpastcss?graymean>rideshotlatesaidroadvar feeljohnrickportfast'UA-deadpoorbilltypeU.S.woodmust2px;Inforankwidewantwalllead[0];paulwavesure$('#waitmassarmsgoesgainlangpaid!-- lockunitrootwalkfirmwifexml"songtest20pxkindrowstoolfontmailsafestarmapscorerainflowbabyspansays4px;6px;artsfootrealwikiheatsteptriporg/lakeweaktoldFormcastfansbankveryrunsjulytask1px;goalgrewslowedgeid="sets5px;.js?40pxif (soonseatnonetubezerosentreedfactintogiftharm18pxcamehillboldzoomvoideasyringfillpeakinitcost3px;jacktagsbitsrolleditknewnearironfreddiskwentsoilputs/js/holyT22:ISBNT20:adamsees