dmzoneill/Synology 108tb limit bypass.txt

## Synology 108tb limit bypass.txt
# Goal: To extend Synology volume beyond the webpage limit (108TB)
# Disks available
# 12 disks x 16TB = 192TB

# Assuming you already have an 8 disk volume
# 8 disks x 16TB = 128TB (Max allowed by Synology)
# lost space due to parity + whatever else ^^

# Synology uses partition raid, not drive raid
# There are 3 raid arrays across matching partitions on each disk

cat /proc/mdstat

> e.g:
> md2 : active raid5 sata1p5[11] sata2p5[10] sata3p5[9] sata4p5[8] sata5p5[11] sata6p5[10]
                     sata7p5[9] sata8p5[8]
		     **all partition 5**
> md1 : active raid1 sata1p2[11] sata2p2[10] sata3p2[9] sata4p2[8] sata5p2[11] sata6p2[10]
                     sata7p2[9] sata8p2[8]
		     **all partition 2**
> md0 : active raid1 sata1p1[11] sata2p1[10] sata3p1[9] sata4p1[8] sata5p1[11] sata6p1[10]
                     sata7p1[9] sata8p1[8]
		     **all partition 1**
> note the partition numbers ^

# Steps
# 1. Add 4 new disks with matching partition structure.
# 2. Fix UUIDs
# 3. Add partitions to the arrays
# 4. Extend filesystem

###############################
######## 1. Add disks #########
###############################

# There are a few ways of doing this
# fdisk > 2.31 supports GPT (synology has 2.33.2)
# So you could use fdisk to eye ball the existing partitions, #
# and create the new partitions on the new drives.
#
# If sfdisk and gdisk were available,
# these would support copying the partition structure from one disk to another.

# I opted for using DD to copy the structure.
#
# Copy the GPT table from 1 of the disks to the 4 new disks
# determine raid type for the partition table size

parted -ms /dev/sata5 print |tail -1|cut -b1
> 5

# do some maths
# (128*5)+1024 = 1664

# copy the table off 1 of the disks

dd if=/dev/sata8 of=GPT_TABLE bs=1 count=1664

# duplicate the partition table to the 4 new disks

dd if=GPT_TABLE of=/dev/sata9 bs=1 count=1664
dd if=GPT_TABLE of=/dev/sata10 bs=1 count=1664
dd if=GPT_TABLE of=/dev/sata11 bs=1 count=1664
dd if=GPT_TABLE of=/dev/sata12 bs=1 count=1664

# fdisk will complain about corrupt backup table

fdisk -l

> The backup GPT table is corrupt, but the primary appears OK, so that will be used.
> Disk /dev/sata9: 14.6 TiB, 16000900661248 bytes, 31251759104 sectors

> Disk /dev/sata10: 14.6 TiB, 16000900661248 bytes, 31251759104 sectors
> Disk model: ST16000NM001G-2KK103

> The backup GPT table is corrupt, but the primary appears OK, so that will be used.
> Disk /dev/sata11: 14.6 TiB, 16000900661248 bytes, 31251759104 sectors

> The backup GPT table is corrupt, but the primary appears OK, so that will be used.
> Disk /dev/sata12: 14.6 TiB, 16000900661248 bytes, 31251759104 sectors

# use fdisk to correct this

fdisk /dev/sata9; w; enter
fdisk /dev/sata10; w; enter
fdisk /dev/sata11; w; enter
fdisk /dev/sata12; w; enter

# We can do a bit of inspection of the GPT backup table if so inclined.
# last 33 sectors of the disk
# Compare old disk and the new disk

dd if=/dev/sata8 of=sata8.img bs=512 skip=31251759070
dd if=/dev/sata10 of=sata10.img bs=512 skip=31251759070

hexdump -C sata8.img

> 00000000  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
> *
> 00000200  0f 88 9d a1 fc 05 3b 4d  a0 06 74 3f 0f 84 91 1e  |......;M..t?....|
> 00000210  27 0c c6 3d 73 14 ea 4f  87 d0 ad 60 fd c8 03 e7  |'..=s..O...`....|
> 00000220  00 20 00 00 00 00 00 00  ff 1f 00 01 00 00 00 00  |. ..............|
> 00000230  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
> *
> 00000280  0f 88 9d a1 fc 05 3b 4d  a0 06 74 3f 0f 84 91 1e  |......;M..t?....|
> 00000290  a1 21 3e c7 68 a9 5d 43  87 70 67 a0 dc 6c ca 6a  |.!>.h.]C.pg..l.j|
> 000002a0  00 20 00 01 00 00 00 00  ff 1f 40 01 00 00 00 00  |. ........@.....|
> 000002b0  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
> *
> 00000400  0f 88 9d a1 fc 05 3b 4d  a0 06 74 3f 0f 84 91 1e  |......;M..t?....|
> 00000410  df 9e f5 3b 71 c6 35 43  81 9f 4a bb 45 af a1 17  |...;q.5C..J.E...|
> 00000420  e0 5e 44 01 00 00 00 00  3f 1c bd 46 07 00 00 00  |.^D.....?..F....|
> 00000430  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
> *
> 00004200  45 46 49 20 50 41 52 54  00 00 01 00 5c 00 00 00  |EFI PART....\...|
> 00004210  f7 e1 8b 85 00 00 00 00  ff ff bf 46 07 00 00 00  |...........F....|
> 00004220  01 00 00 00 00 00 00 00  22 00 00 00 00 00 00 00  |........".......|
> 00004230  de ff bf 46 07 00 00 00  ea db 0f 82 d8 a0 11 4d  |...F...........M|
> 00004240  8a 1f d7 7f 1d 6d 53 fe  df ff bf 46 07 00 00 00  |.....mS....F....|
> 00004250  80 00 00 00 80 00 00 00  26 42 2a e3 00 00 00 00  |........&B*.....|
> 00004260  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
> *
> 00004400


hexdump -C sata10.img

> 00000000  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
> *
> 00000200  0f 88 9d a1 fc 05 3b 4d  a0 06 74 3f 0f 84 91 1e  |......;M..t?....|
> 00000210  8c 3b ab db d9 17 5e 4e  b7 fd c6 78 05 84 5a c8  |.;....^N...x..Z.|
> 00000220  00 20 00 00 00 00 00 00  ff 1f 00 01 00 00 00 00  |. ..............|
> 00000230  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
> *
> 00000280  0f 88 9d a1 fc 05 3b 4d  a0 06 74 3f 0f 84 91 1e  |......;M..t?....|
> 00000290  2d 55 ca 1f 98 2e 1a 4b  8f 8c b9 63 03 bf a6 23  |-U.....K...c...#|
> 000002a0  00 20 00 01 00 00 00 00  ff 1f 40 01 00 00 00 00  |. ........@.....|
> 000002b0  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
> *
> 00000400  0f 88 9d a1 fc 05 3b 4d  a0 06 74 3f 0f 84 91 1e  |......;M..t?....|
> 00000410  32 08 48 a2 c9 92 f8 4a  83 a5 22 07 24 d0 2a 0b  |2.H....J..".$.*.|
> 00000420  e0 5e 44 01 00 00 00 00  3f 1c bd 46 07 00 00 00  |.^D.....?..F....|
> 00000430  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
> *
> 00004200  45 46 49 20 50 41 52 54  00 00 01 00 5c 00 00 00  |EFI PART....\...|
> 00004210  b6 ca 57 50 00 00 00 00  ff ff bf 46 07 00 00 00  |..WP.......F....|
> 00004220  01 00 00 00 00 00 00 00  22 00 00 00 00 00 00 00  |........".......|
> 00004230  de ff bf 46 07 00 00 00  37 76 2a ea 8f 6f 54 40  |...F....7v*..oT@|
> 00004240  ab 53 36 d3 1c 18 8f 53  df ff bf 46 07 00 00 00  |.S6....S...F....|
> 00004250  80 00 00 00 80 00 00 00  d7 ae 73 b3 00 00 00 00  |..........s.....|
> 00004260  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
> *
> 00004400
>

###############################
######## 2. Fix UUIDs #########
###############################

# Reviewing the UUIDs we can now see we have duplicates.

blkid

> /dev/sata9: PTUUID="8dfdf8a5-a209-4fd9-8716-f949e7c313b5" PTTYPE="gpt"
> /dev/sata10: PTUUID="8dfdf8a5-a209-4fd9-8716-f949e7c313b5" PTTYPE="gpt"
> /dev/sata11: PTUUID="8dfdf8a5-a209-4fd9-8716-f949e7c313b5" PTTYPE="gpt"
> /dev/sata12: PTUUID="8dfdf8a5-a209-4fd9-8716-f949e7c313b5" PTTYPE="gpt"
> ...

# Depending on the FS, you would naturally go for tune2fs or btrfstune to fix the UUIDs.
# However, Synology have removed these tools.
#
# so, something like this won't work
# for X in 9 10 11 12; do
# btrfstune -U $(uuidgen) /dev/sata${X}p1
# btrfstune -U $(uuidgen) /dev/sata${X}p2
# btrfstune -U $(uuidgen) /dev/sata${X}p5
# done

# 1 method for fixing this is

fdisk /dev/sata9

> Welcome to fdisk (util-linux 2.33.2).
> Changes will remain in memory only, until you decide to write them.
> Be careful before using the write command.

>Command (m for help): x
> Expert command (m for help): i
> Enter new disk UUID (in 8-4-4-4-12 format): 6849bb56-a5c7-4e11-8878-b0ca0ca181fd
> Disk identifier changed from 8DFDF8A5-A209-4FD9-8716-F949E7C313B5 to 6849BB56-A5C7-4E11-8878-B0CA0CA181FD.

> Expert command (m for help): u
> Partition number (1,2,5, default 5): 1
> New UUID (in 8-4-4-4-12 format): b11ff16c-90b8-4d45-9f43-a70b3800781d
> Partition UUID changed from 9ED624C6-9079-4E23-B208-FD068DF75325 to B11FF16C-90B8-4D45-9F43-A70B3800781D.

> Expert command (m for help): u
> Partition number (1,2,5, default 5): 2
> New UUID (in 8-4-4-4-12 format): 76bbd5fc-e13f-4ba3-b2bd-a7a05d0ed262
> Partition UUID changed from E58E0427-C0A6-4AEB-AF87-D0592C854940 to 76BBD5FC-E13F-4BA3-B2BD-A7A05D0ED262.

> Expert command (m for help): u
> Partition number (1,2,5, default 5): 5
> New UUID (in 8-4-4-4-12 format): 0ebc64a3-b66a-40dc-82cc-c8a570f2b272
> Partition UUID changed from 52DB2F94-0363-442C-89FE-BA7FF951B818 to 0EBC64A3-B66A-40DC-82CC-C8A570F2B272.

> Expert command (m for help): r
> Command (m for help): w
> The partition table has been altered.
> Calling ioctl() to re-read partition table.
> Syncing disks.

# repeat for each disk, using uuidgen for generating unique UUIDs.

##################################################
######## 3. Add partitions to the arrays #########
##################################################

# CHeck the current state

cat /proc/mdstat
> Personalities : [raid1] [raid6] [raid5] [raid4] [raidF1]
> md2 : active raid5 sata8p5[7] sata7p5[6] sata1p5[0] sata6p5[5] sata5p5[4] sata4p5[3] sata3p5[2] sata2p5[1]
>      109306083712 blocks super 1.2 level 5, 64k chunk, algorithm 2 [8/8] [UUUUUUUU]
>
> md1 : active raid1 sata8p2[7] sata7p2[6] sata1p2[0] sata6p2[5] sata2p2[4] sata3p2[3] sata4p2[2] sata5p2[1]
>      2097088 blocks [12/8] [UUUUUUUU____]
>
> md0 : active raid1 sata8p1[7] sata7p1[6] sata1p1[0] sata6p1[5] sata2p1[4] sata3p1[3] sata4p1[2] sata5p1[1]
>      2490176 blocks [12/8] [UUUUUUUU____]

# lets add the 4 new drives
# Add the partitions to their appropriate array

mdadm --add /dev/md2 /dev/sata9p5 /dev/sata10p5 /dev/sata11p5 /dev/sata12p5
> mdadm: added /dev/sata9p5
> mdadm: added /dev/sata10p5
> mdadm: added /dev/sata11p5
> mdadm: added /dev/sata12p5

# Add the partitions to their appropriate array

mdadm --add /dev/md1 /dev/sata9p2 /dev/sata10p2 /dev/sata11p2 /dev/sata12p2
> mdadm: added /dev/sata9p2
> mdadm: added /dev/sata10p2
> mdadm: added /dev/sata11p2
> mdadm: added /dev/sata12p2

# Add the partitions to their appropriate array

mdadm --add /dev/md0 /dev/sata9p1 /dev/sata10p1 /dev/sata11p1 /dev/sata12p1
> mdadm: added /dev/sata9p1
> mdadm: added /dev/sata10p1
> mdadm: added /dev/sata11p1
> mdadm: added /dev/sata12p1

# Grow the raid5,6,data array
mdadm --grow --raid-devices=12 /dev/md2

# note there is no growing the raid1 arrays (mirror)
# the array will now start the reshaping

cat /proc/mdstat
> Personalities : [raid1] [raid6] [raid5] [raid4] [raidF1]
> md2 : active raid5 sata12p5[11] sata11p5[10] sata10p5[9] sata9p5[8] sata8p5[7] sata7p5[6] sata1p5[0] sata6p5[5] sata5p5[4] > sata4p5[3] sata3p5[2] sata2p5[1]
>      109306083712 blocks super 1.2 level 5, 64k chunk, algorithm 2 [12/12] [UUUUUUUUUUUU]
>      [>....................]  reshape =  0.0% (913840/15615154816) finish=4556.2min speed=57115K/sec

# Force check the arrays

cat /sys/block/md0/md/sync_action
> idle

cat /sys/block/md1/md/sync_action
> idle

cat /sys/block/md2/md/sync_action
> reshape

echo check > /sys/block/md0/md/sync_action
cat /sys/block/md0/md/sync_action
> check
cat /sys/block/md0/md/sync_action
> idle

echo check > /sys/block/md1/md/sync_action
cat /sys/block/md1/md/sync_action
> check
cat /sys/block/md1/md/sync_action
> idle

dmesg

> [3855582.034575] md: data-check of RAID array md0
> [3855643.368651] md: md0: data-check done.
> [3855643.399737] md: md0: current auto_remap = 0
> [3855660.722242] md: data-check of RAID array md1
> [3855712.139841] md: md1: data-check done.
> [3855712.208568] md: md1: current auto_remap = 0


mdadm --detail /dev/md2

> /dev/md2:
>         Version : 1.2
>   Creation Time : Tue Dec  6 18:16:50 2022
>      Raid Level : raid5
>      Array Size : 109306083712 (104242.40 GiB 111929.43 GB)
>   Used Dev Size : 15615154816 (14891.77 GiB 15989.92 GB)
>    Raid Devices : 12
>   Total Devices : 12
>     Persistence : Superblock is persistent
>
>     Update Time : Mon Jan 23 11:42:02 2023
>           State : active, reshaping
>  Active Devices : 12
> Working Devices : 12
>  Failed Devices : 0
>   Spare Devices : 0
>
>          Layout : left-symmetric
>      Chunk Size : 64K
>
>  Reshape Status : 60% complete
>   Delta Devices : 4, (8->12)
>
>            Name : synology2:2  (local to host synology2)
>            UUID : 12c6307f:87954e4e:fd9f3130:33c985db
>          Events : 60720

>     Number   Major   Minor   RaidDevice State
>        0       8        5        0      active sync   /dev/sata1p5
>        1       8       21        1      active sync   /dev/sata2p5
>        2       8       37        2      active sync   /dev/sata3p5
>        3       8       53        3      active sync   /dev/sata4p5
>        4       8       69        4      active sync   /dev/sata5p5
>        5       8       85        5      active sync   /dev/sata6p5
>        7       8      117        6      active sync   /dev/sata8p5
>        6       8      101        7      active sync   /dev/sata7p5
>       11       8      181        8      active sync   /dev/sata12p5
>       10       8      165        9      active sync   /dev/sata11p5
>        9       8      149       10      active sync   /dev/sata10p5
>        8       8      133       11      active sync   /dev/sata9p5

# Some addtional checks

for X in {1..12}; do sudo mdadm --examine /dev/sata"$X"p1; done
for X in {1..12}; do sudo mdadm --examine /dev/sata"$X"p2; done
for X in {1..12}; do sudo mdadm --examine /dev/sata"$X"p5; done

###########################################
######## 4. Extend the filesystem #########
###########################################

# Wait for the reshape to complete before extending

df -h /

ash-4.4# df -h /volume1/
Filesystem              Size  Used Avail Use% Mounted on
/dev/mapper/cachedev_0   98T   57T   41T  59% /volume1


btrfs filesystem resize max /volume1
df -h /
btrfs fi us /volume1/
btrfs fi us -T /volume1/
btrfs filesystem du /btr
btrfs filesystem usage /volume1/
btrfs filesystem show /volume1/

# Btrfs uses "blocks" to store data.
# Blocks may not be filled completely
# to recover free space, inspect usage and recover if necessary

btrfs fi df /volume1/
> Data, single: total=56.85TiB, used=56.83TiB
> System, DUP: total=32.00MiB, used=640.00KiB
> Metadata, DUP: total=76.00GiB, used=62.06GiB
> GlobalReserve, single: total=2.00GiB, used=0.00B

for i in 0 5 10 15 20 25 30 40 50 60 70 80 90 100
do
	echo "${0}: Running with ${i}%"
	btrfs balance start -dusage=$i -musage=$i /volume1/
done

################################
######## miscellaneous #########
################################

# Attempt to fix a broken array

mdadm --assemble --force /dev/md127 /dev/sdb1 /dev/sdc1 /dev/sdd1

# Readd a temporarily removed drive
# check the events diff

for X in {1..12}; do
   mdadm --examine /dev/sata"$X"p5 | egrep 'Event|/dev/sata'
done

/dev/sata1p5:
         Events : 992655
/dev/sata2p5:
         Events : 992655
/dev/sata3p5:
         Events : 992655
/dev/sata4p5:
         Events : 992655
/dev/sata5p5:
         Events : 158
/dev/sata6p5:
         Events : 992655
/dev/sata7p5:
         Events : 992655
/dev/sata8p5:
         Events : 992655
/dev/sata9p5:
         Events : 992655
/dev/sata10p5:
         Events : 992655
/dev/sata11p5:
         Events : 992655
/dev/sata12p5:
         Events : 992655

# If the events are not too far apart you can try to add with

mdadm: --add for /dev/sataX to dev/md2

# if that deosn't work, we will have to do a full reshaping

mdadm: --re-add for /dev/sataX to dev/md2
	# Goal: To extend Synology volume beyond the webpage limit (108TB)
	# Disks available
	# 12 disks x 16TB = 192TB

	# Assuming you already have an 8 disk volume
	# 8 disks x 16TB = 128TB (Max allowed by Synology)
	# lost space due to parity + whatever else ^^

	# Synology uses partition raid, not drive raid
	# There are 3 raid arrays across matching partitions on each disk

	cat /proc/mdstat

	> e.g:
	> md2 : active raid5 sata1p5[11] sata2p5[10] sata3p5[9] sata4p5[8] sata5p5[11] sata6p5[10]
	sata7p5[9] sata8p5[8]
	all partition 5
	> md1 : active raid1 sata1p2[11] sata2p2[10] sata3p2[9] sata4p2[8] sata5p2[11] sata6p2[10]
	sata7p2[9] sata8p2[8]
	all partition 2
	> md0 : active raid1 sata1p1[11] sata2p1[10] sata3p1[9] sata4p1[8] sata5p1[11] sata6p1[10]
	sata7p1[9] sata8p1[8]
	all partition 1
	> note the partition numbers ^

	# Steps
	# 1. Add 4 new disks with matching partition structure.
	# 2. Fix UUIDs
	# 3. Add partitions to the arrays
	# 4. Extend filesystem

	###############################
	######## 1. Add disks #########
	###############################

	# There are a few ways of doing this
	# fdisk > 2.31 supports GPT (synology has 2.33.2)
	# So you could use fdisk to eye ball the existing partitions, #
	# and create the new partitions on the new drives.
	#
	# If sfdisk and gdisk were available,
	# these would support copying the partition structure from one disk to another.

	# I opted for using DD to copy the structure.
	#
	# Copy the GPT table from 1 of the disks to the 4 new disks
	# determine raid type for the partition table size

	parted -ms /dev/sata5 print \|tail -1\|cut -b1
	> 5

	# do some maths
	# (128*5)+1024 = 1664

	# copy the table off 1 of the disks

	dd if=/dev/sata8 of=GPT_TABLE bs=1 count=1664

	# duplicate the partition table to the 4 new disks

	dd if=GPT_TABLE of=/dev/sata9 bs=1 count=1664
	dd if=GPT_TABLE of=/dev/sata10 bs=1 count=1664
	dd if=GPT_TABLE of=/dev/sata11 bs=1 count=1664
	dd if=GPT_TABLE of=/dev/sata12 bs=1 count=1664

	# fdisk will complain about corrupt backup table

	fdisk -l

	> The backup GPT table is corrupt, but the primary appears OK, so that will be used.
	> Disk /dev/sata9: 14.6 TiB, 16000900661248 bytes, 31251759104 sectors

	> Disk /dev/sata10: 14.6 TiB, 16000900661248 bytes, 31251759104 sectors
	> Disk model: ST16000NM001G-2KK103

	> The backup GPT table is corrupt, but the primary appears OK, so that will be used.
	> Disk /dev/sata11: 14.6 TiB, 16000900661248 bytes, 31251759104 sectors

	> The backup GPT table is corrupt, but the primary appears OK, so that will be used.
	> Disk /dev/sata12: 14.6 TiB, 16000900661248 bytes, 31251759104 sectors

	# use fdisk to correct this

	fdisk /dev/sata9; w; enter
	fdisk /dev/sata10; w; enter
	fdisk /dev/sata11; w; enter
	fdisk /dev/sata12; w; enter

	# We can do a bit of inspection of the GPT backup table if so inclined.
	# last 33 sectors of the disk
	# Compare old disk and the new disk

	dd if=/dev/sata8 of=sata8.img bs=512 skip=31251759070
	dd if=/dev/sata10 of=sata10.img bs=512 skip=31251759070

	hexdump -C sata8.img

	> 00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \|................\|
	> *
	> 00000200 0f 88 9d a1 fc 05 3b 4d a0 06 74 3f 0f 84 91 1e \|......;M..t?....\|
	> 00000210 27 0c c6 3d 73 14 ea 4f 87 d0 ad 60 fd c8 03 e7 \|'..=s..O...`....\|
	> 00000220 00 20 00 00 00 00 00 00 ff 1f 00 01 00 00 00 00 \|. ..............\|
	> 00000230 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \|................\|
	> *
	> 00000280 0f 88 9d a1 fc 05 3b 4d a0 06 74 3f 0f 84 91 1e \|......;M..t?....\|
	> 00000290 a1 21 3e c7 68 a9 5d 43 87 70 67 a0 dc 6c ca 6a \|.!>.h.]C.pg..l.j\|
	> 000002a0 00 20 00 01 00 00 00 00 ff 1f 40 01 00 00 00 00 \|. ........@.....\|
	> 000002b0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \|................\|
	> *
	> 00000400 0f 88 9d a1 fc 05 3b 4d a0 06 74 3f 0f 84 91 1e \|......;M..t?....\|
	> 00000410 df 9e f5 3b 71 c6 35 43 81 9f 4a bb 45 af a1 17 \|...;q.5C..J.E...\|
	> 00000420 e0 5e 44 01 00 00 00 00 3f 1c bd 46 07 00 00 00 \|.^D.....?..F....\|
	> 00000430 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \|................\|
	> *
	> 00004200 45 46 49 20 50 41 52 54 00 00 01 00 5c 00 00 00 \|EFI PART....\...\|
	> 00004210 f7 e1 8b 85 00 00 00 00 ff ff bf 46 07 00 00 00 \|...........F....\|
	> 00004220 01 00 00 00 00 00 00 00 22 00 00 00 00 00 00 00 \|........".......\|
	> 00004230 de ff bf 46 07 00 00 00 ea db 0f 82 d8 a0 11 4d \|...F...........M\|
	> 00004240 8a 1f d7 7f 1d 6d 53 fe df ff bf 46 07 00 00 00 \|.....mS....F....\|
	> 00004250 80 00 00 00 80 00 00 00 26 42 2a e3 00 00 00 00 \|........&B*.....\|
	> 00004260 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \|................\|
	> *
	> 00004400


	hexdump -C sata10.img

	> 00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \|................\|
	> *
	> 00000200 0f 88 9d a1 fc 05 3b 4d a0 06 74 3f 0f 84 91 1e \|......;M..t?....\|
	> 00000210 8c 3b ab db d9 17 5e 4e b7 fd c6 78 05 84 5a c8 \|.;....^N...x..Z.\|
	> 00000220 00 20 00 00 00 00 00 00 ff 1f 00 01 00 00 00 00 \|. ..............\|
	> 00000230 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \|................\|
	> *
	> 00000280 0f 88 9d a1 fc 05 3b 4d a0 06 74 3f 0f 84 91 1e \|......;M..t?....\|
	> 00000290 2d 55 ca 1f 98 2e 1a 4b 8f 8c b9 63 03 bf a6 23 \|-U.....K...c...#\|
	> 000002a0 00 20 00 01 00 00 00 00 ff 1f 40 01 00 00 00 00 \|. ........@.....\|
	> 000002b0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \|................\|
	> *
	> 00000400 0f 88 9d a1 fc 05 3b 4d a0 06 74 3f 0f 84 91 1e \|......;M..t?....\|
	> 00000410 32 08 48 a2 c9 92 f8 4a 83 a5 22 07 24 d0 2a 0b \|2.H....J..".$.*.\|
	> 00000420 e0 5e 44 01 00 00 00 00 3f 1c bd 46 07 00 00 00 \|.^D.....?..F....\|
	> 00000430 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \|................\|
	> *
	> 00004200 45 46 49 20 50 41 52 54 00 00 01 00 5c 00 00 00 \|EFI PART....\...\|
	> 00004210 b6 ca 57 50 00 00 00 00 ff ff bf 46 07 00 00 00 \|..WP.......F....\|
	> 00004220 01 00 00 00 00 00 00 00 22 00 00 00 00 00 00 00 \|........".......\|
	> 00004230 de ff bf 46 07 00 00 00 37 76 2a ea 8f 6f 54 40 \|...F....7v*..oT@\|
	> 00004240 ab 53 36 d3 1c 18 8f 53 df ff bf 46 07 00 00 00 \|.S6....S...F....\|
	> 00004250 80 00 00 00 80 00 00 00 d7 ae 73 b3 00 00 00 00 \|..........s.....\|
	> 00004260 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 \|................\|
	> *
	> 00004400
	>

	###############################
	######## 2. Fix UUIDs #########
	###############################

	# Reviewing the UUIDs we can now see we have duplicates.

	blkid

	> /dev/sata9: PTUUID="8dfdf8a5-a209-4fd9-8716-f949e7c313b5" PTTYPE="gpt"
	> /dev/sata10: PTUUID="8dfdf8a5-a209-4fd9-8716-f949e7c313b5" PTTYPE="gpt"
	> /dev/sata11: PTUUID="8dfdf8a5-a209-4fd9-8716-f949e7c313b5" PTTYPE="gpt"
	> /dev/sata12: PTUUID="8dfdf8a5-a209-4fd9-8716-f949e7c313b5" PTTYPE="gpt"
	> ...

	# Depending on the FS, you would naturally go for tune2fs or btrfstune to fix the UUIDs.
	# However, Synology have removed these tools.
	#
	# so, something like this won't work
	# for X in 9 10 11 12; do
	# btrfstune -U $(uuidgen) /dev/sata${X}p1
	# btrfstune -U $(uuidgen) /dev/sata${X}p2
	# btrfstune -U $(uuidgen) /dev/sata${X}p5
	# done

	# 1 method for fixing this is

	fdisk /dev/sata9

	> Welcome to fdisk (util-linux 2.33.2).
	> Changes will remain in memory only, until you decide to write them.
	> Be careful before using the write command.

	>Command (m for help): x
	> Expert command (m for help): i
	> Enter new disk UUID (in 8-4-4-4-12 format): 6849bb56-a5c7-4e11-8878-b0ca0ca181fd
	> Disk identifier changed from 8DFDF8A5-A209-4FD9-8716-F949E7C313B5 to 6849BB56-A5C7-4E11-8878-B0CA0CA181FD.

	> Expert command (m for help): u
	> Partition number (1,2,5, default 5): 1
	> New UUID (in 8-4-4-4-12 format): b11ff16c-90b8-4d45-9f43-a70b3800781d
	> Partition UUID changed from 9ED624C6-9079-4E23-B208-FD068DF75325 to B11FF16C-90B8-4D45-9F43-A70B3800781D.

	> Expert command (m for help): u
	> Partition number (1,2,5, default 5): 2
	> New UUID (in 8-4-4-4-12 format): 76bbd5fc-e13f-4ba3-b2bd-a7a05d0ed262
	> Partition UUID changed from E58E0427-C0A6-4AEB-AF87-D0592C854940 to 76BBD5FC-E13F-4BA3-B2BD-A7A05D0ED262.

	> Expert command (m for help): u
	> Partition number (1,2,5, default 5): 5
	> New UUID (in 8-4-4-4-12 format): 0ebc64a3-b66a-40dc-82cc-c8a570f2b272
	> Partition UUID changed from 52DB2F94-0363-442C-89FE-BA7FF951B818 to 0EBC64A3-B66A-40DC-82CC-C8A570F2B272.

	> Expert command (m for help): r
	> Command (m for help): w
	> The partition table has been altered.
	> Calling ioctl() to re-read partition table.
	> Syncing disks.

	# repeat for each disk, using uuidgen for generating unique UUIDs.

	##################################################
	######## 3. Add partitions to the arrays #########
	##################################################

	# CHeck the current state

	cat /proc/mdstat
	> Personalities : [raid1] [raid6] [raid5] [raid4] [raidF1]
	> md2 : active raid5 sata8p5[7] sata7p5[6] sata1p5[0] sata6p5[5] sata5p5[4] sata4p5[3] sata3p5[2] sata2p5[1]
	> 109306083712 blocks super 1.2 level 5, 64k chunk, algorithm 2 [8/8] [UUUUUUUU]
	>
	> md1 : active raid1 sata8p2[7] sata7p2[6] sata1p2[0] sata6p2[5] sata2p2[4] sata3p2[3] sata4p2[2] sata5p2[1]
	> 2097088 blocks [12/8] [UUUUUUUU____]
	>
	> md0 : active raid1 sata8p1[7] sata7p1[6] sata1p1[0] sata6p1[5] sata2p1[4] sata3p1[3] sata4p1[2] sata5p1[1]
	> 2490176 blocks [12/8] [UUUUUUUU____]

	# lets add the 4 new drives
	# Add the partitions to their appropriate array

	mdadm --add /dev/md2 /dev/sata9p5 /dev/sata10p5 /dev/sata11p5 /dev/sata12p5
	> mdadm: added /dev/sata9p5
	> mdadm: added /dev/sata10p5
	> mdadm: added /dev/sata11p5
	> mdadm: added /dev/sata12p5

	# Add the partitions to their appropriate array

	mdadm --add /dev/md1 /dev/sata9p2 /dev/sata10p2 /dev/sata11p2 /dev/sata12p2
	> mdadm: added /dev/sata9p2
	> mdadm: added /dev/sata10p2
	> mdadm: added /dev/sata11p2
	> mdadm: added /dev/sata12p2

	# Add the partitions to their appropriate array

	mdadm --add /dev/md0 /dev/sata9p1 /dev/sata10p1 /dev/sata11p1 /dev/sata12p1
	> mdadm: added /dev/sata9p1
	> mdadm: added /dev/sata10p1
	> mdadm: added /dev/sata11p1
	> mdadm: added /dev/sata12p1

	# Grow the raid5,6,data array
	mdadm --grow --raid-devices=12 /dev/md2

	# note there is no growing the raid1 arrays (mirror)
	# the array will now start the reshaping

	cat /proc/mdstat
	> Personalities : [raid1] [raid6] [raid5] [raid4] [raidF1]
	> md2 : active raid5 sata12p5[11] sata11p5[10] sata10p5[9] sata9p5[8] sata8p5[7] sata7p5[6] sata1p5[0] sata6p5[5] sata5p5[4] > sata4p5[3] sata3p5[2] sata2p5[1]
	> 109306083712 blocks super 1.2 level 5, 64k chunk, algorithm 2 [12/12] [UUUUUUUUUUUU]
	> [>....................] reshape = 0.0% (913840/15615154816) finish=4556.2min speed=57115K/sec

	# Force check the arrays

	cat /sys/block/md0/md/sync_action
	> idle

	cat /sys/block/md1/md/sync_action
	> idle

	cat /sys/block/md2/md/sync_action
	> reshape

	echo check > /sys/block/md0/md/sync_action
	cat /sys/block/md0/md/sync_action
	> check
	cat /sys/block/md0/md/sync_action
	> idle

	echo check > /sys/block/md1/md/sync_action
	cat /sys/block/md1/md/sync_action
	> check
	cat /sys/block/md1/md/sync_action
	> idle

	dmesg

	> [3855582.034575] md: data-check of RAID array md0
	> [3855643.368651] md: md0: data-check done.
	> [3855643.399737] md: md0: current auto_remap = 0
	> [3855660.722242] md: data-check of RAID array md1
	> [3855712.139841] md: md1: data-check done.
	> [3855712.208568] md: md1: current auto_remap = 0


	mdadm --detail /dev/md2

	> /dev/md2:
	> Version : 1.2
	> Creation Time : Tue Dec 6 18:16:50 2022
	> Raid Level : raid5
	> Array Size : 109306083712 (104242.40 GiB 111929.43 GB)
	> Used Dev Size : 15615154816 (14891.77 GiB 15989.92 GB)
	> Raid Devices : 12
	> Total Devices : 12
	> Persistence : Superblock is persistent
	>
	> Update Time : Mon Jan 23 11:42:02 2023
	> State : active, reshaping
	> Active Devices : 12
	> Working Devices : 12
	> Failed Devices : 0
	> Spare Devices : 0
	>
	> Layout : left-symmetric
	> Chunk Size : 64K
	>
	> Reshape Status : 60% complete
	> Delta Devices : 4, (8->12)
	>
	> Name : synology2:2 (local to host synology2)
	> UUID : 12c6307f:87954e4e:fd9f3130:33c985db
	> Events : 60720

	> Number Major Minor RaidDevice State
	> 0 8 5 0 active sync /dev/sata1p5
	> 1 8 21 1 active sync /dev/sata2p5
	> 2 8 37 2 active sync /dev/sata3p5
	> 3 8 53 3 active sync /dev/sata4p5
	> 4 8 69 4 active sync /dev/sata5p5
	> 5 8 85 5 active sync /dev/sata6p5
	> 7 8 117 6 active sync /dev/sata8p5
	> 6 8 101 7 active sync /dev/sata7p5
	> 11 8 181 8 active sync /dev/sata12p5
	> 10 8 165 9 active sync /dev/sata11p5
	> 9 8 149 10 active sync /dev/sata10p5
	> 8 8 133 11 active sync /dev/sata9p5

	# Some addtional checks

	for X in {1..12}; do sudo mdadm --examine /dev/sata"$X"p1; done
	for X in {1..12}; do sudo mdadm --examine /dev/sata"$X"p2; done
	for X in {1..12}; do sudo mdadm --examine /dev/sata"$X"p5; done

	###########################################
	######## 4. Extend the filesystem #########
	###########################################

	# Wait for the reshape to complete before extending

	df -h /

	ash-4.4# df -h /volume1/
	Filesystem Size Used Avail Use% Mounted on
	/dev/mapper/cachedev_0 98T 57T 41T 59% /volume1


	btrfs filesystem resize max /volume1
	df -h /
	btrfs fi us /volume1/
	btrfs fi us -T /volume1/
	btrfs filesystem du /btr
	btrfs filesystem usage /volume1/
	btrfs filesystem show /volume1/

	# Btrfs uses "blocks" to store data.
	# Blocks may not be filled completely
	# to recover free space, inspect usage and recover if necessary

	btrfs fi df /volume1/
	> Data, single: total=56.85TiB, used=56.83TiB
	> System, DUP: total=32.00MiB, used=640.00KiB
	> Metadata, DUP: total=76.00GiB, used=62.06GiB
	> GlobalReserve, single: total=2.00GiB, used=0.00B

	for i in 0 5 10 15 20 25 30 40 50 60 70 80 90 100
	do
	echo "${0}: Running with ${i}%"
	btrfs balance start -dusage=$i -musage=$i /volume1/
	done

	################################
	######## miscellaneous #########
	################################

	# Attempt to fix a broken array

	mdadm --assemble --force /dev/md127 /dev/sdb1 /dev/sdc1 /dev/sdd1

	# Readd a temporarily removed drive
	# check the events diff

	for X in {1..12}; do
	mdadm --examine /dev/sata"$X"p5 \| egrep 'Event\|/dev/sata'
	done

	/dev/sata1p5:
	Events : 992655
	/dev/sata2p5:
	Events : 992655
	/dev/sata3p5:
	Events : 992655
	/dev/sata4p5:
	Events : 992655
	/dev/sata5p5:
	Events : 158
	/dev/sata6p5:
	Events : 992655
	/dev/sata7p5:
	Events : 992655
	/dev/sata8p5:
	Events : 992655
	/dev/sata9p5:
	Events : 992655
	/dev/sata10p5:
	Events : 992655
	/dev/sata11p5:
	Events : 992655
	/dev/sata12p5:
	Events : 992655

	# If the events are not too far apart you can try to add with

	mdadm: --add for /dev/sataX to dev/md2

	# if that deosn't work, we will have to do a full reshaping

	mdadm: --re-add for /dev/sataX to dev/md2