#!/bin/sh -vex                                                                                                                                                                                                       

# PROBLEM: when I do "zfs send | ssh backup-server zfs recv",                                                                                                                                                        
#          I see that "zfs send" finishes after about 15 minutes, but                                                                                                                                                
#          "zfs recv" on the remote host consumes 100% of one CPU core for 10 HOURS.                                                                                                                                 
#          It does eventually exit.                                                                                                                                                                                  
#                                                                                                                                                                                                                    
#          The delay appears to be proportional to the number of datasets in the pool, so                                                                                                                            
#          on a system with 10 users, I didn't notice the problem, but                                                                                                                                               
#          on a system with 1000 users, I noticed right away!                                                                                                                                                        
#          (On both systems, there are separate datasets for each user's $HOME and $MAIL.)                                                                                                                           
#                                                                                                                                                                                                                    
# 05:01 <PMT> it's pretty simple, just, "recv_fix_encryption_hierarchy does horrifically inefficient lookups that could be either cached or refactored, and it bottlenecks super badly"                              

# Report what we're running.                                                                                                                                                                                         
zfs --version
uname -a

# First clean up anything from a previous attempt.                                                                                                                                                                   
zpool export A || :
rm -f /A.vdev

# Make a new pool with 1000 datasets in it.                                                                                                                                                                          
truncate -s1T /A.vdev
zpool create -O canmount=off -O mountpoint=none A /A.vdev
python3 -c 'print(open("/dev/urandom", "rb").read(32).hex(),file=open("/A.passphrase", "w"))'
zfs create -v -o encryption=on -o keyformat=hex -o keylocation=file:///A.passphrase A/A
seq 100 999 >/A.txt
time xargs --arg-file=/A.txt -I N zfs create -v A/A/N

# Make a snapshot and then send it to ourselves.                                                                                                                                                                     
# Normally this would be a pipeline "zfs send ⋯ A/A@1 | ssh B zfs recv ⋯ B/A", but                                                                                                                                   
# use a temporary file so we can clearly see how long each step takes.                                                                                                                                               
zfs snapshot -r A/A@1
>/[email protected] time zfs send --large-block --embed --compressed --raw --replicate A/A@1
</[email protected] time zfs receive -v -o readonly=on -o canmount=noauto A/B

# This happens with incrementals also.                                                                                                                                                                               
zfs snapshot -r A/A@2
>/[email protected] time zfs send --large-block --embed --compressed --raw --replicate -I 1 A/A@2
</[email protected] time zfs receive -v -o readonly=on -o canmount=noauto -F A/B