ToS for control plane; trust cryptographically secure checksums

* part two; prev. commit: 185be3fb6027bdc * up cli Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Dec 22, 2024 · 79434c6 · 79434c6
1 parent 8185be3
commit 79434c6
Show file tree

Hide file tree

Showing 7 changed files with 40 additions and 27 deletions.
diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli
 go 1.23.2
 
 require (
-	github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1
+	github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602
 	github.com/fatih/color v1.18.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo/v2 v2.21.0

diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum
@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
-github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1 h1:SO4oqmdxqvhi/unWsN7SJsejbh/M7P3TlXXEo3evrLw=
-github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw=
+github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602 h1:Ypfy9sLjPti1CcebiPGFadXObWDPNPrE+gkAX902Sm8=
+github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

diff --git a/cmd/ishard/go.mod b/cmd/ishard/go.mod
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/ishard
 go 1.23.2
 
 require (
-	github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1
+	github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602
 	github.com/json-iterator/go v1.1.12
 	github.com/vbauerster/mpb/v4 v4.12.2
 )

diff --git a/cmd/ishard/go.sum b/cmd/ishard/go.sum
@@ -1,6 +1,6 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
-github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1 h1:SO4oqmdxqvhi/unWsN7SJsejbh/M7P3TlXXEo3evrLw=
-github.com/NVIDIA/aistore v1.3.26-0.20241220002133-3661cee7b7e1/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw=
+github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602 h1:Ypfy9sLjPti1CcebiPGFadXObWDPNPrE+gkAX902Sm8=
+github.com/NVIDIA/aistore v1.3.26-0.20241222164523-8185be3fb602/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=

diff --git a/cmn/cos/cksum.go b/cmn/cos/cksum.go
@@ -20,11 +20,14 @@ import (
 	jsoniter "github.com/json-iterator/go"
 )
 
-// NOTE: not supporting SHA-3 family is its current golang.org/x/crypto/sha3 source
-//       doesn't implement BinaryMarshaler & BinaryUnmarshaler interfaces
-//       (see also https://golang.org/pkg/encoding)
+// [NOTE]
+// - currently, we have only two crypto-secure types: sha256 and sha512
+// - see related object comparison logic in cmn/objattrs
 
-// checksums
+// [TODO]
+// revisit and maybe add SHA-3 family (see golang.org/x/crypto/sha3 for: `BinaryMarshaler`)
+
+// supported checksums
 const (
 	ChecksumNone   = "none"
 	ChecksumXXHash = "xxhash"

diff --git a/cmn/objattrs.go b/cmn/objattrs.go
@@ -14,6 +14,7 @@ import (
 	"github.com/NVIDIA/aistore/api/apc"
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
+	"github.com/NVIDIA/aistore/cmn/feat"
 )
 
 // LOM custom metadata stored under `lomCustomMD`.
@@ -280,19 +281,28 @@ func (oa *ObjAttrs) CheckEq(rem cos.OAH) error {
 	}
 
 	// checksum check
-	if a, b := rem.Checksum(), oa.Cksum; !a.IsEmpty() && !b.IsEmpty() && a.Ty() == b.Ty() {
-		if !a.Equal(b) {
-			return fmt.Errorf("%s checksum %s != %s remote", a.Ty(), b, a)
-		}
-		cksumVal = a.Val()
-		//
-		// NOTE: including xxhash in trusted checksums
-		//
-		switch a.Ty() {
-		case cos.ChecksumXXHash, cos.ChecksumSHA256, cos.ChecksumSHA512:
-			sameCksum = true
+	if a, b := rem.Checksum(), oa.Cksum; a != nil && b != nil {
+		cksumType := a.Ty()
+		if !a.IsEmpty() && !b.IsEmpty() && cksumType == b.Ty() {
+			if !a.Equal(b) {
+				return fmt.Errorf("%s checksum %s != %s remote", cksumType, b, a)
+			}
+			cksumVal = a.Val()
+
+			// [NOTE]
+			// unless overridden via feature flag
+			// trust two checksums, namely md5 and xxhash, that are _not_ cryptographically secure
+
+			switch {
+			case Rom.Features().IsSet(feat.TrustCryptoSafeChecksums):
+				sameCksum = (cksumType == cos.ChecksumSHA256 || cksumType == cos.ChecksumSHA512)
+			default:
+				debug.Assert(cksumType != cos.ChecksumNone)
+				sameCksum = cksumType != cos.ChecksumCRC32C
+			}
+
+			count++
 		}
-		count++
 	}
 
 	// custom MD: ETag check (ignoring enclosing quotes)

diff --git a/docs/feature_flags.md b/docs/feature_flags.md
@@ -51,11 +51,11 @@ By default, all features are disabled, and the corresponding 64-bit field is set
 ```console
 $ ais config cluster features <TAB-TAB>
 
-Enforce-IntraCluster-Access           Fsync-PUT                             Ignore-LimitedCoexistence-Conflicts
-Skip-Loading-VersionChecksum-MD       LZ4-Block-1MB                         S3-Presigned-Request
-Do-not-Auto-Detect-FileShare          LZ4-Frame-Checksum                    Do-not-Optimize-Listing-Virtual-Dirs
-S3-API-via-Root                       Do-not-Allow-Passing-FQN-to-ETL         Disable-Cold-GET
-S3-Reverse-Proxy                      none
+Enforce-IntraCluster-Access            LZ4-Block-1MB                          Do-not-Optimize-Listing-Virtual-Dirs   Do-not-Delete-When-Rebalancing
+Skip-Loading-VersionChecksum-MD        LZ4-Frame-Checksum                     Disable-Cold-GET                       Do-not-Set-Control-Plane-ToS
+Do-not-Auto-Detect-FileShare           Do-not-Allow-Passing-FQN-to-ETL        Streaming-Cold-GET                     none
+S3-API-via-Root                        Ignore-LimitedCoexistence-Conflicts    S3-Reverse-Proxy
+Fsync-PUT                              S3-Presigned-Request                   S3-Use-Path-Style
 ```
 
 For example: