Encrypted FreeBSD - Full ZFS & EFI Mirror - Cisco UCS C220-M3

JBOD Mode on RAID Cisco UCS C220-M3 //  FreeBSD 11.2

root@:~ # gpart create -s gpt mfisyspd0
root@:~ # gpart create -s gpt mfisyspd1

Create EFI partition

root@:~ # gpart add -s 1m -t efi -l efi0 -a 4k mfisyspd0
root@:~ # gpart add -s 1m -t efi -l efi1 -a 4k mfisyspd1

Create Boot Partition (Unencrypted & using ZFS)

root@:~ # gpart add -s 512m -t freebsd-zfs -lboot0 -a 4k mfisyspd0
root@:~ # gpart add -s 512m -t freebsd-zfs -lboot1 -a 4k mfisyspd1

Create ZFS data Partition (Encrypted & using ZFS)

root@:~ # gpart add  -t freebsd-zfs -ldisk0 -a 4k mfisyspd0
root@:~ # gpart add -t freebsd-zfs -ldisk1 -a 4k mfisyspd1

Mirror EFI partition using gmirror & perpare for EFI boot

root@:~ # kldload geom_mirror
root@:~ # gmirror label -v  -b round-robin efi  gpt/efi0 gpt/efi1
root@:~ # newfs_msdos /dev/mirror/efi
root@:~ # mount_msdosfs  /dev/mirror/efi /mnt
root@:~ # mkdir -p /mnt/efi/boot
root@:~ # cp /boot/boot1.efi /mnt/efi/boot/BOOTx64.efi
root@:~ # echo BOOTx64.efi > /mnt/efi/boot/STARTUP.NSH
root@:~ # umount /mnt/

Prepare Encrypted Partition for Encrypted Zpool :

root@:~ # dd if=/dev/random of=/tmp/zroot.key bs=64 count=1
root@:~ # geli init -P -s 4096 -K /tmp/zroot.key -b -g gpt/disk0
root@:~ # geli init -P -s 4096 -K /tmp/zroot.key -b -g gpt/disk1
root@:~ # geli attach -p -k /tmp/zroot.key gpt/disk0
root@:~ # geli attach -p -k /tmp/zroot.key gpt/disk1

Zpool on Encrypted Partition :

root@:~ # kldload zfs
root@:~ # zpool create -R /mnt -o cachefile=/tmp/zpool.cache -O mountpoint=/ -O atime=off -O canmount=off -O compression=on zpool mirror gpt/disk0.eli gpt/disk1.eli
root@:~ # zfs create -o canmount=off -o mountpoint=none zpool/ROOT
root@:~ # zfs create -o mountpoint=/ zpool/ROOT/default
root@:~ # zfs create -o canmount=off zpool/usr
root@:~ # zfs create zpool/usr/local
root@:~ # zfs create -o canmount=off zpool/var
root@:~ # zfs create zpool/var/log

Zboot on ZFS and Non-Encrypted Partition :

root@:~ # zpool create -R /mnt -o cachefile=/tmp/zpool.cache -O mountpoint=/zboot  -O atime=off -O compression=on boot mirror gpt/boot0 gpt/boot1

Manual Installing FreeBSD 

root@:~ # cd /mnt
root@:/mnt # tar xvJpf /usr/freebsd-dist/base.txz
root@:/mnt # tar xvJpf /usr/freebsd-dist/kernel.txz 

Move /mnt/boot to /zboot/boot & link /mnt/zboot/boot to /mnt/boot

root@:/mnt # mv boot/ zboot/
root@:/mnt # ln -s zboot/boot/ 

Using chroot edit system configuration:

root@:/mnt #chroot /mnt
 root@:/ #passwd root 

Important!!

root@:/  #vi /boot/loader.conf 

zfs_load="YES"
aesni_load="YES"
geom_eli_load="YES"
geom_eli_passphrase_prompt="NO"
geom_mirror_load="YES"
geli_mfisyspd0p3_keyfile_load="YES"
geli_mfisyspd0p3_keyfile_type="mfisyspd0p3:geli_keyfile"
geli_mfisyspd0p3_keyfile_name="/boot/keys/zroot.key"
geli_mfisyspd1p3_keyfile_load="YES"
geli_mfisyspd1p3_keyfile_type="mfisyspd1p3:geli_keyfile"
geli_mfisyspd1p3_keyfile_name="/boot/keys/zroot.key"
zpool_cache_load="YES"
zpool_cache_name="/boot/zfs/zpool.cache"
zpool_cache_type="/boot/zfs/zpool.cache"
vfs.root.mountfrom="zfs:zpool/ROOT/default"

root@:/  #vi /etc/rc.conf

ifconfig_igb1="DHCP"
zfs_enable="YES"
clear_tmp_enable="YES"
syslogd_flags="-ss"
sendmail_enable="NONE"
hostname="lithium"
sshd_enable="YES"

Important !!

Exit from chroot and copy zpool.cache and zroot.key

root@:/ # exit
root@:~ # cp /tmp/zpool.cache /mnt/zboot/boot/zfs/
root@:~ # mkdir /mnt/zboot/boot/keys
root@:~ # cp /tmp/zroot.key /mnt/zboot/boot/keys/

Don't forget to backup zroot.key & zpool.cache on your pendisk

done 

reboot