From b89781dd7ea862f5a1e1d36d443525b9dd7ea529 Mon Sep 17 00:00:00 2001 From: Alex Aizman Date: Wed, 8 Jan 2025 14:44:44 -0500 Subject: [PATCH] core: remove "file name too long" limitation * part three, prev. commit: 6603f24351a3ef Signed-off-by: Alex Aizman --- ais/daemon.go | 2 +- ais/earlystart.go | 2 +- ais/htrun.go | 2 +- ais/proxy.go | 2 +- ais/prxauth.go | 2 +- ais/test/bucket_test.go | 2 +- ais/test/common_test.go | 2 +- ais/test/dsort_test.go | 2 +- ais/test/ec_test.go | 2 +- ais/test/objprops_test.go | 2 +- ais/test/promote_test.go | 2 +- ais/test/s3/e2e_s3_test.go | 2 +- ais/test/s3_compat_test.go | 2 +- ais/tgtobj.go | 5 ++-- ais/utils.go | 2 +- api/authn/loadtoken.go | 2 +- bench/tools/aisloader/run.go | 2 +- cmd/authn/hserv.go | 2 +- cmd/authn/main.go | 2 +- cmd/authn/mgr.go | 2 +- cmd/cli/cli/auth_hdlr.go | 2 +- cmd/cli/cli/err.go | 2 +- cmd/cli/cli/etl.go | 2 +- cmd/cli/cli/get.go | 2 +- cmd/cli/cli/init.go | 2 +- cmd/cli/cli/job_hdlr.go | 2 +- cmd/cli/cli/ls.go | 2 +- cmd/cli/cli/multiobj.go | 2 +- cmd/cli/cli/object.go | 2 +- cmd/cli/cli/smap.go | 2 +- cmd/cli/cli/verbfobj.go | 5 ++++ cmd/cli/cli/yap.go | 2 +- cmd/cli/config/config.go | 2 +- cmd/cli/go.mod | 2 +- cmd/cli/go.sum | 4 +-- cmd/cli/teb/mountpath.go | 2 +- cmd/ishard/go.mod | 2 +- cmd/ishard/go.sum | 4 +-- cmn/client.go | 2 +- cmn/cos/err.go | 2 +- cmn/k8s/client.go | 2 +- cmn/k8s/k8s.go | 2 +- cmn/tests/http_test.go | 2 +- core/lcopy.go | 12 ++++++-- core/lfile.go | 57 +++++++++++++++++++++++++++++++----- core/lom.go | 22 +++++++++----- ext/dload/api.go | 2 +- tools/e2e.go | 2 +- tools/init.go | 2 +- xact/xs/wi_lso.go | 8 +++-- 50 files changed, 133 insertions(+), 68 deletions(-) diff --git a/ais/daemon.go b/ais/daemon.go index d77efe0a650..61baff86ed7 100644 --- a/ais/daemon.go +++ b/ais/daemon.go @@ -1,6 +1,6 @@ // Package ais provides core functionality for the AIStore object storage. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package ais diff --git a/ais/earlystart.go b/ais/earlystart.go index b2ae095bf94..7169db85aea 100644 --- a/ais/earlystart.go +++ b/ais/earlystart.go @@ -1,6 +1,6 @@ // Package ais provides core functionality for the AIStore object storage. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package ais diff --git a/ais/htrun.go b/ais/htrun.go index c74f4a00c0f..7e8f88657d4 100644 --- a/ais/htrun.go +++ b/ais/htrun.go @@ -1,6 +1,6 @@ // Package ais provides core functionality for the AIStore object storage. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package ais diff --git a/ais/proxy.go b/ais/proxy.go index e06f183a6a2..90f9c3e9f89 100644 --- a/ais/proxy.go +++ b/ais/proxy.go @@ -1,6 +1,6 @@ // Package ais provides core functionality for the AIStore object storage. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package ais diff --git a/ais/prxauth.go b/ais/prxauth.go index 95cc410ad68..9bd4c5af10b 100644 --- a/ais/prxauth.go +++ b/ais/prxauth.go @@ -1,6 +1,6 @@ // Package ais provides core functionality for the AIStore object storage. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package ais diff --git a/ais/test/bucket_test.go b/ais/test/bucket_test.go index 69aa079b001..00a37a7ca63 100644 --- a/ais/test/bucket_test.go +++ b/ais/test/bucket_test.go @@ -1,6 +1,6 @@ // Package integration_test. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package integration_test diff --git a/ais/test/common_test.go b/ais/test/common_test.go index bcffc0a3513..1f7a6f3a283 100644 --- a/ais/test/common_test.go +++ b/ais/test/common_test.go @@ -1,6 +1,6 @@ // Package integration_test. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package integration_test diff --git a/ais/test/dsort_test.go b/ais/test/dsort_test.go index 5fb83e7be47..78bce9fb597 100644 --- a/ais/test/dsort_test.go +++ b/ais/test/dsort_test.go @@ -1,7 +1,7 @@ //nolint:dupl // copy-paste benign and can wait // Package integration_test. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package integration_test diff --git a/ais/test/ec_test.go b/ais/test/ec_test.go index fa976a24ffc..99c71d26a8e 100644 --- a/ais/test/ec_test.go +++ b/ais/test/ec_test.go @@ -1,6 +1,6 @@ // Package integration_test. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package integration_test diff --git a/ais/test/objprops_test.go b/ais/test/objprops_test.go index 8e5a9a483b6..9eabfae1a99 100644 --- a/ais/test/objprops_test.go +++ b/ais/test/objprops_test.go @@ -1,6 +1,6 @@ // Package integration_test. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package integration_test diff --git a/ais/test/promote_test.go b/ais/test/promote_test.go index a75c66e918e..641f63eb3e2 100644 --- a/ais/test/promote_test.go +++ b/ais/test/promote_test.go @@ -1,6 +1,6 @@ // Package integration_test. /* - * Copyright (c) 2022=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved. */ package integration_test diff --git a/ais/test/s3/e2e_s3_test.go b/ais/test/s3/e2e_s3_test.go index ac6e58c1da8..b9af37c5b18 100644 --- a/ais/test/s3/e2e_s3_test.go +++ b/ais/test/s3/e2e_s3_test.go @@ -1,6 +1,6 @@ // Package s3_integration provides tests of compatibility with AWS S3 /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package s3_test diff --git a/ais/test/s3_compat_test.go b/ais/test/s3_compat_test.go index 329acaa20c7..7c12c922e38 100644 --- a/ais/test/s3_compat_test.go +++ b/ais/test/s3_compat_test.go @@ -1,6 +1,6 @@ // Package integration_test. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package integration_test diff --git a/ais/tgtobj.go b/ais/tgtobj.go index 37cd0135e33..1f866c8d21f 100644 --- a/ais/tgtobj.go +++ b/ais/tgtobj.go @@ -179,7 +179,7 @@ func (poi *putOI) do(resphdr http.Header, r *http.Request, dpq *dpq) (int, error func (poi *putOI) putObject() (ecode int, err error) { if lom := poi.lom; lom.IsFntl() { - // override wfqn and shorten (!) lom, both + // fixup fntl var ( short = lom.ShortenFntl() saved = lom.PushFntl(short) @@ -1021,8 +1021,7 @@ func (goi *getOI) txfini() (ecode int, err error) { fqn = goi.lom.LBGet() // best-effort GET load balancing (see also mirror.findLeastUtilized()) } // open - // TODO -- FIXME: use lom.Open() instead of os.Open(); TestECChecksum - lmfh, err = os.Open(fqn) + lmfh, err = goi.lom.OpenFile() if err != nil { if os.IsNotExist(err) { // NOTE: retry only once and only when ec-enabled - see goi.restoreFromAny() diff --git a/ais/utils.go b/ais/utils.go index d063bb38e19..c6e89f4de1f 100644 --- a/ais/utils.go +++ b/ais/utils.go @@ -1,6 +1,6 @@ // Package ais provides core functionality for the AIStore object storage. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package ais diff --git a/api/authn/loadtoken.go b/api/authn/loadtoken.go index e75c778242d..465d7c75c20 100644 --- a/api/authn/loadtoken.go +++ b/api/authn/loadtoken.go @@ -1,6 +1,6 @@ // Package authn provides AuthN API over HTTP(S) /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package authn diff --git a/bench/tools/aisloader/run.go b/bench/tools/aisloader/run.go index f374e3f907d..a97f876e409 100644 --- a/bench/tools/aisloader/run.go +++ b/bench/tools/aisloader/run.go @@ -1,6 +1,6 @@ // Package aisloader /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ // AIS loader (aisloader) is a tool to measure storage performance. It's a load diff --git a/cmd/authn/hserv.go b/cmd/authn/hserv.go index f9ab52a4b82..786e71b0ee6 100644 --- a/cmd/authn/hserv.go +++ b/cmd/authn/hserv.go @@ -1,6 +1,6 @@ // Package authn is authentication server for AIStore. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package main diff --git a/cmd/authn/main.go b/cmd/authn/main.go index f8c5607857d..d66b56ff2dd 100644 --- a/cmd/authn/main.go +++ b/cmd/authn/main.go @@ -1,6 +1,6 @@ // Package authn is authentication server for AIStore. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package main diff --git a/cmd/authn/mgr.go b/cmd/authn/mgr.go index d7acc9de59a..7f556186900 100644 --- a/cmd/authn/mgr.go +++ b/cmd/authn/mgr.go @@ -1,6 +1,6 @@ // Package authn is authentication server for AIStore. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package main diff --git a/cmd/cli/cli/auth_hdlr.go b/cmd/cli/cli/auth_hdlr.go index bada5902273..199f45afeca 100644 --- a/cmd/cli/cli/auth_hdlr.go +++ b/cmd/cli/cli/auth_hdlr.go @@ -1,7 +1,7 @@ // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters. // This file handles commands that create entities in the cluster. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cli diff --git a/cmd/cli/cli/err.go b/cmd/cli/cli/err.go index 012cb21b059..66f0324a3b1 100644 --- a/cmd/cli/cli/err.go +++ b/cmd/cli/cli/err.go @@ -1,7 +1,7 @@ // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters. // This file contains error handlers and utilities. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cli diff --git a/cmd/cli/cli/etl.go b/cmd/cli/cli/etl.go index e59ebedf416..b93d110a19e 100644 --- a/cmd/cli/cli/etl.go +++ b/cmd/cli/cli/etl.go @@ -1,7 +1,7 @@ // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters. // This file handles commands that control running jobs in the cluster. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cli diff --git a/cmd/cli/cli/get.go b/cmd/cli/cli/get.go index cbe72fba451..30102203be5 100644 --- a/cmd/cli/cli/get.go +++ b/cmd/cli/cli/get.go @@ -1,7 +1,7 @@ // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters. // This file handles object operations. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cli diff --git a/cmd/cli/cli/init.go b/cmd/cli/cli/init.go index b3e9273cff4..5254c0b9428 100644 --- a/cmd/cli/cli/init.go +++ b/cmd/cli/cli/init.go @@ -1,6 +1,6 @@ // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cli diff --git a/cmd/cli/cli/job_hdlr.go b/cmd/cli/cli/job_hdlr.go index 6ce4d622ce5..688d097aceb 100644 --- a/cmd/cli/cli/job_hdlr.go +++ b/cmd/cli/cli/job_hdlr.go @@ -1,7 +1,7 @@ // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters. // This file handles commands that control running jobs in the cluster. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cli diff --git a/cmd/cli/cli/ls.go b/cmd/cli/cli/ls.go index 40dbfca82b1..5fc02c9e225 100644 --- a/cmd/cli/cli/ls.go +++ b/cmd/cli/cli/ls.go @@ -1,6 +1,6 @@ // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cli diff --git a/cmd/cli/cli/multiobj.go b/cmd/cli/cli/multiobj.go index 3ebe40afcc1..dcaceca1429 100644 --- a/cmd/cli/cli/multiobj.go +++ b/cmd/cli/cli/multiobj.go @@ -1,7 +1,7 @@ // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters. // This file handles object operations. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cli diff --git a/cmd/cli/cli/object.go b/cmd/cli/cli/object.go index 34448e7ab62..c94b46d25f5 100644 --- a/cmd/cli/cli/object.go +++ b/cmd/cli/cli/object.go @@ -1,7 +1,7 @@ // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters. // This file handles object operations. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cli diff --git a/cmd/cli/cli/smap.go b/cmd/cli/cli/smap.go index 3844b298bb2..294c729fb15 100644 --- a/cmd/cli/cli/smap.go +++ b/cmd/cli/cli/smap.go @@ -1,7 +1,7 @@ // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters. // This file contains util functions and types. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cli diff --git a/cmd/cli/cli/verbfobj.go b/cmd/cli/cli/verbfobj.go index db2306ac89b..076b2a0464e 100644 --- a/cmd/cli/cli/verbfobj.go +++ b/cmd/cli/cli/verbfobj.go @@ -452,6 +452,11 @@ func putRegular(c *cli.Context, bck cmn.Bck, objName, path string, finfo os.File } break } + + if e, ok := err.(*cmn.ErrCreateHreq); ok { + return e + } + e := stripErr(err) if i < iters-1 { s := fmt.Sprintf("[#%d] %s: %v - retrying...", i+1, path, e) diff --git a/cmd/cli/cli/yap.go b/cmd/cli/cli/yap.go index b0fadcbc986..c2b082447a5 100644 --- a/cmd/cli/cli/yap.go +++ b/cmd/cli/cli/yap.go @@ -1,6 +1,6 @@ // Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cli diff --git a/cmd/cli/config/config.go b/cmd/cli/config/config.go index af078638a8c..00158af9158 100644 --- a/cmd/cli/config/config.go +++ b/cmd/cli/config/config.go @@ -1,6 +1,6 @@ // Package config provides types and functions to configure AIS CLI. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package config diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod index f24c21467d9..fee7b164e6c 100644 --- a/cmd/cli/go.mod +++ b/cmd/cli/go.mod @@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli go 1.23.2 require ( - github.com/NVIDIA/aistore v1.3.26-0.20250106161132-d8ae4afc3594 + github.com/NVIDIA/aistore v1.3.26-0.20250108155940-8c676ef07b6d github.com/fatih/color v1.18.0 github.com/json-iterator/go v1.1.12 github.com/onsi/ginkgo/v2 v2.21.0 diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum index ae9c429a814..250c4d19a58 100644 --- a/cmd/cli/go.sum +++ b/cmd/cli/go.sum @@ -1,7 +1,7 @@ code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/NVIDIA/aistore v1.3.26-0.20250106161132-d8ae4afc3594 h1:Qn1yaBunXuJOSSIC82e9o2YrPi932T8zmlpZmn5hWPU= -github.com/NVIDIA/aistore v1.3.26-0.20250106161132-d8ae4afc3594/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw= +github.com/NVIDIA/aistore v1.3.26-0.20250108155940-8c676ef07b6d h1:dEWTFT5KVO32GAw68uGfiz4WqOwPP5XVps8l5KZOqa0= +github.com/NVIDIA/aistore v1.3.26-0.20250108155940-8c676ef07b6d/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= diff --git a/cmd/cli/teb/mountpath.go b/cmd/cli/teb/mountpath.go index 2a40e9eef43..2ec069de2c8 100644 --- a/cmd/cli/teb/mountpath.go +++ b/cmd/cli/teb/mountpath.go @@ -1,6 +1,6 @@ // Package teb contains templates and (templated) tables to format CLI output. /* - * Copyright (c) 2023=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved. */ package teb diff --git a/cmd/ishard/go.mod b/cmd/ishard/go.mod index 459c338b1a4..2bb60ef0282 100644 --- a/cmd/ishard/go.mod +++ b/cmd/ishard/go.mod @@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/ishard go 1.23.2 require ( - github.com/NVIDIA/aistore v1.3.26-0.20250106161132-d8ae4afc3594 + github.com/NVIDIA/aistore v1.3.26-0.20250108155940-8c676ef07b6d github.com/json-iterator/go v1.1.12 github.com/vbauerster/mpb/v4 v4.12.2 ) diff --git a/cmd/ishard/go.sum b/cmd/ishard/go.sum index f6e60dda654..b94f5844b51 100644 --- a/cmd/ishard/go.sum +++ b/cmd/ishard/go.sum @@ -1,6 +1,6 @@ code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= -github.com/NVIDIA/aistore v1.3.26-0.20250106161132-d8ae4afc3594 h1:Qn1yaBunXuJOSSIC82e9o2YrPi932T8zmlpZmn5hWPU= -github.com/NVIDIA/aistore v1.3.26-0.20250106161132-d8ae4afc3594/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw= +github.com/NVIDIA/aistore v1.3.26-0.20250108155940-8c676ef07b6d h1:dEWTFT5KVO32GAw68uGfiz4WqOwPP5XVps8l5KZOqa0= +github.com/NVIDIA/aistore v1.3.26-0.20250108155940-8c676ef07b6d/go.mod h1:mjhY9OGIZULaC79+iRfzEUvUZw7aIWklJ8um321QVpw= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= diff --git a/cmn/client.go b/cmn/client.go index adf771fc1db..acac0ce121b 100644 --- a/cmn/client.go +++ b/cmn/client.go @@ -1,7 +1,7 @@ // Package cmn provides common constants, types, and utilities for AIS clients // and AIStore. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package cmn diff --git a/cmn/cos/err.go b/cmn/cos/err.go index f5691146cbe..2af08d69afa 100644 --- a/cmn/cos/err.go +++ b/cmn/cos/err.go @@ -167,7 +167,7 @@ func IsPathErr(err error) (ok bool) { // "file name too long" errno 0x24 (36); either one of the two possible reasons: // - len(pathname) > PATH_MAX = 4096 // - len(basename) > 255 -func IsFntl(err error) bool { +func IsErrFntl(err error) bool { return strings.Contains(err.Error(), "too long") && errors.Is(err, syscall.ENAMETOOLONG) } diff --git a/cmn/k8s/client.go b/cmn/k8s/client.go index 84f7e682ece..c155af52f90 100644 --- a/cmn/k8s/client.go +++ b/cmn/k8s/client.go @@ -1,6 +1,6 @@ // Package k8s: initialization, client, and misc. helpers /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package k8s diff --git a/cmn/k8s/k8s.go b/cmn/k8s/k8s.go index d8563cb1a28..45662a8c28f 100644 --- a/cmn/k8s/k8s.go +++ b/cmn/k8s/k8s.go @@ -1,6 +1,6 @@ // Package k8s: initialization, client, and misc. helpers /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package k8s diff --git a/cmn/tests/http_test.go b/cmn/tests/http_test.go index 0933f2ea752..ccb78fcecf2 100644 --- a/cmn/tests/http_test.go +++ b/cmn/tests/http_test.go @@ -1,6 +1,6 @@ // Package test provides tests for common low-level types and utilities for all aistore projects /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package tests_test diff --git a/core/lcopy.go b/core/lcopy.go index 49257e4c842..bbd8ffc79ee 100644 --- a/core/lcopy.go +++ b/core/lcopy.go @@ -227,8 +227,9 @@ func (lom *LOM) Copy(mi *fs.Mountpath, buf []byte) (err error) { copyFQN = mi.MakePathFQN(lom.Bucket(), fs.ObjectType, lom.ObjName) workFQN = mi.MakePathFQN(lom.Bucket(), fs.WorkfileType, fs.WorkfileCopy+"."+lom.ObjName) ) - // check if the copy destination exists and then skip copying if it's also identical - if errExists := cos.Stat(copyFQN); errExists == nil { + // copy is a no-op if the destination exists and is identical + errExists := cos.Stat(copyFQN) + if errExists == nil { cplom := AllocLOM(lom.ObjName) defer FreeLOM(cplom) if errExists = cplom.InitFQN(copyFQN, lom.Bucket()); errExists == nil { @@ -238,6 +239,13 @@ func (lom *LOM) Copy(mi *fs.Mountpath, buf []byte) (err error) { } } } + } else if cos.IsErrFntl(errExists) { + // fixup fntl + short := lom.ShortenFntl() + saved := lom.PushFntl(short) + defer lom.PopFntl(saved) + copyFQN = mi.MakePathFQN(lom.Bucket(), fs.ObjectType, lom.ObjName) + workFQN = mi.MakePathFQN(lom.Bucket(), fs.WorkfileType, fs.WorkfileCopy+"."+lom.ObjName) } // copy diff --git a/core/lfile.go b/core/lfile.go index ffbb92da99b..f8bd96bfaf3 100644 --- a/core/lfile.go +++ b/core/lfile.go @@ -8,11 +8,13 @@ import ( "fmt" "os" "path/filepath" + rdebug "runtime/debug" "syscall" "github.com/NVIDIA/aistore/cmn" "github.com/NVIDIA/aistore/cmn/cos" "github.com/NVIDIA/aistore/cmn/debug" + "github.com/NVIDIA/aistore/cmn/nlog" ) const ( @@ -36,15 +38,35 @@ func (e *errBdir) Error() string { // open // +// open read-only, return os.File +func (lom *LOM) OpenFile() (fh *os.File, _ error) { + reader, err := lom.Open() + if err != nil { + return nil, err + } + fh = reader.(*os.File) + return fh, nil +} + +// same as above but return reader func (lom *LOM) Open() (fh cos.LomReader, err error) { fh, err = os.Open(lom.FQN) - if err == nil || !os.IsNotExist(err) { - return fh, err - } - if e := lom._checkBdir(); e != nil { - return nil, e + switch { + case err == nil: + return fh, nil + case os.IsNotExist(err): + if e := lom._checkBdir(); e != nil { + err = e + } + return nil, err + default: + // DEBUG + if cos.IsErrFntl(err) { + nlog.Errorln(">>>", err) + rdebug.PrintStack() + } + return nil, err } - return nil, err } // @@ -58,7 +80,7 @@ func (lom *LOM) Create() (cos.LomWriter, error) { func (lom *LOM) CreateWork(wfqn string) (cos.LomWriter, error) { return lom._cf(wfqn) } // -> lom func (lom *LOM) CreatePart(wfqn string) (*os.File, error) { return lom._cf(wfqn) } // TODO: differentiate -func (lom *LOM) CreateSlice(wfqn string) (*os.File, error) { return lom._cf(wfqn) } // TODO: ditto +func (lom *LOM) CreateSlice(wfqn string) (*os.File, error) { return lom._cf(wfqn) } // --/-- func (lom *LOM) _cf(fqn string) (fh *os.File, err error) { fh, err = os.OpenFile(fqn, _openFlags, cos.PermRWR) @@ -66,7 +88,26 @@ func (lom *LOM) _cf(fqn string) (fh *os.File, err error) { return fh, nil } if !os.IsNotExist(err) { - // TODO: cos.CheckMvToVirtDir(err, fqn) + if cos.IsErrFntl(err) { + // - when creating LOM: fixup fntl in place + // - otherwise, return fntl error (with an implied requirement that caller must handle it) + if fqn != lom.FQN { + return nil, err + } + var ( + short = lom.ShortenFntl() + saved = lom.PushFntl(short) + ) + fh, err = os.OpenFile(short[0], _openFlags, cos.PermRWR) + if err == nil { + lom.md.lid = lom.md.lid.setlmfl(lmflFntl) + lom.SetCustomKey(cmn.OrigFntl, saved[0]) + } else { + debug.Assert(!cos.IsErrFntl(err)) + lom.PopFntl(saved) + } + return fh, err + } T.FSHC(err, lom.Mountpath(), "") return nil, err } diff --git a/core/lom.go b/core/lom.go index 0eaa25f9eb2..e18c70bf3ce 100644 --- a/core/lom.go +++ b/core/lom.go @@ -474,7 +474,7 @@ func (lom *LOM) LoadUnsafe() (err error) { // either a) handled or b) benign from the caller's perspective if _, err = lom.lmfs(true); err == nil { if lom.bid() == 0 { - // ditto (MetaverLOM = 1) + // ditto MetaverLOM = 1 lom.setbid(lom.Bprops().BID) } err = lom._checkBucket(bmd) @@ -562,7 +562,7 @@ func (lom *LOM) FromFS() error { // e.g. err "stat .../aaa/111: not a directory" when there's existing ".../aaa" object case strings.Contains(err.Error(), "not a directory") && cos.IsPathErr(err): return fmt.Errorf("%w (object in the path?)", err) - case cos.IsFntl(err): + case cos.IsErrFntl(err): lom.md.lid = lomBID(lom.Bprops().BID) lom.md.lid = lom.md.lid.setlmfl(lmflFntl) @@ -570,12 +570,12 @@ func (lom *LOM) FromFS() error { short := lom.ShortenFntl() saved := lom.PushFntl(short) size, atimefs, _, err = lom.Fstat(true) - lom.PopFntl(saved) - if err == nil { goto exist } - debug.Assert(!cos.IsFntl(err)) + + lom.PopFntl(saved) + debug.Assert(!cos.IsErrFntl(err)) if os.IsNotExist(err) { return err } @@ -664,10 +664,16 @@ func (lom *LOM) Unlock(exclusive bool) { // const ( - prefixFntl = ".%" + prefixFntl = ".x" ) -func (lom *LOM) IsFntl() bool { return lom.md.lid.haslmfl(lmflFntl) } +func (lom *LOM) IsFntl() bool { + return lom.md.lid.haslmfl(lmflFntl) +} + +func (lom *LOM) HasFntlPrefix() bool { + return strings.HasPrefix(lom.ObjName, prefixFntl) +} func (lom *LOM) ShortenFntl() []string { noname := prefixFntl + cos.ChecksumB2S(cos.UnsafeB(lom.FQN), cos.ChecksumSHA256) @@ -695,9 +701,11 @@ func (lom *LOM) OrigFntl() []string { func (lom *LOM) PushFntl(temp []string) (saved []string) { saved = []string{lom.FQN, lom.ObjName} lom.FQN, lom.ObjName = temp[0], temp[1] + lom.HrwFQN = &lom.FQN return saved } func (lom *LOM) PopFntl(saved []string) { lom.FQN, lom.ObjName = saved[0], saved[1] + lom.HrwFQN = &lom.FQN } diff --git a/ext/dload/api.go b/ext/dload/api.go index 099407af831..21990675674 100644 --- a/ext/dload/api.go +++ b/ext/dload/api.go @@ -1,6 +1,6 @@ // Package dload implements functionality to download resources into AIS cluster from external source. /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package dload diff --git a/tools/e2e.go b/tools/e2e.go index 0d71ec25534..bc854e83635 100644 --- a/tools/e2e.go +++ b/tools/e2e.go @@ -1,6 +1,6 @@ // Package tools provides common tools and utilities for all unit and integration tests /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package tools diff --git a/tools/init.go b/tools/init.go index 70bcbc4a7ed..978258aec7e 100644 --- a/tools/init.go +++ b/tools/init.go @@ -1,6 +1,6 @@ // Package tools provides common tools and utilities for all unit and integration tests /* - * Copyright (c) 2018=2025, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package tools diff --git a/xact/xs/wi_lso.go b/xact/xs/wi_lso.go index 4c5446b3457..31db472d397 100644 --- a/xact/xs/wi_lso.go +++ b/xact/xs/wi_lso.go @@ -134,7 +134,7 @@ func (wi *walkInfo) callback(fqn string, de fs.DirEntry) (entry *cmn.LsoEnt, err lom := core.AllocLOM("") entry, err = wi._cb(lom, fqn) core.FreeLOM(lom) - return + return entry, err } func (wi *walkInfo) _cb(lom *core.LOM, fqn string) (*cmn.LsoEnt, error) { @@ -162,7 +162,7 @@ func (wi *walkInfo) _cb(lom *core.LOM, fqn string) (*cmn.LsoEnt, error) { } // shortcut #1: name-only optimizes-out loading md (NOTE: won't show misplaced and copies) - if wi.msg.IsFlagSet(apc.LsNameOnly) { + if wi.msg.IsFlagSet(apc.LsNameOnly) && !lom.HasFntlPrefix() { if !isOK(status) { return nil, nil } @@ -184,6 +184,10 @@ func (wi *walkInfo) _cb(lom *core.LOM, fqn string) (*cmn.LsoEnt, error) { } if !wi.msg.IsFlagSet(apc.LsMissing) { + if lom.IsFntl() { + status = apc.LocOK // NOTE: fntl never misplaced + return wi.ls(lom, status), nil + } return nil, nil } if local {