aboutsummaryrefslogtreecommitdiff
path: root/bin/ak-fs-add
diff options
context:
space:
mode:
authorkaotisk <kaotisk@arching-kaos.org>2024-03-10 02:08:24 +0200
committerkaotisk <kaotisk@arching-kaos.org>2024-03-10 02:08:24 +0200
commit798a8a50445f894f7e6979b9e688639005d0a5d9 (patch)
treec47570e1a0f1ae3f3b0b4ef404a67137e1e92c25 /bin/ak-fs-add
parentcfdeb61252a100d660c047a4ea1eb5a38bba705b (diff)
downloadarching-kaos-tools-798a8a50445f894f7e6979b9e688639005d0a5d9.tar.gz
arching-kaos-tools-798a8a50445f894f7e6979b9e688639005d0a5d9.tar.bz2
arching-kaos-tools-798a8a50445f894f7e6979b9e688639005d0a5d9.zip
Renamed ak-sm-merkle-tree* to ak-fs-*
renamed: bin/ak-sm-merkle-tree -> bin/ak-fs-add renamed: bin/ak-sm-merkle-tree-to-file -> bin/ak-fs-get Also, minor fixes in README
Diffstat (limited to 'bin/ak-fs-add')
-rwxr-xr-xbin/ak-fs-add226
1 files changed, 226 insertions, 0 deletions
diff --git a/bin/ak-fs-add b/bin/ak-fs-add
new file mode 100755
index 0000000..2ba5e91
--- /dev/null
+++ b/bin/ak-fs-add
@@ -0,0 +1,226 @@
+#!/bin/bash
+#
+# The concept is bit more complicated now
+#
+# 1. For a given file we split in 4KB files inside a temporary directory
+#
+# 2. We then create a map file and a merkle tree containing the resulted files
+# and their sha512sum.
+#
+# 3. We move the splitted files to our $CHKDIR named after their checksums
+#
+# 4. We move the merkle tree pairs to $MERKLEDIR
+#
+# We ultimately want to be seeding the file so
+#
+# 4. We append the checksum of the original file with its name into the map file
+#
+# 5. We rename the map file after its checksum and move it to maps directory
+#
+# 6. We are done!
+#
+
+# Uncomment next line if you want to debug
+# set -xe
+PROGRAM="$(basename $0)"
+
+if [ ! -f "$1" ]
+then
+ echo "ERROR File not found"
+ exit 1
+else
+ base64 $1 > file
+ FILE="file"
+fi
+
+# The directory where the chunked data will be living at
+CHKDIR="$AK_WORKDIR/ftr/"
+# The directory for the map files so we can reconstruct the file
+FILEMAPSDIR="$AK_WORKDIR/fmp/"
+# Merkle tree file/references
+MERKLEDIR="$AK_WORKDIR/fmrk/"
+# A temporary root dir to work on
+TEMPORARYDIR="$(ak-tempassin)"
+# A subdir to split the files there
+TECHDIR="$TEMPORARYDIR/chks/"
+# A pin point to return from where we came from
+CURRENTDIR="$(pwd)/"
+
+# Our snippet for logging debug info
+logit(){
+ ak-logthis "$PROGRAM" "$1" "$2"
+}
+
+# Checking directories and create them if necessary
+rm -rf $TEMPORARYDIR
+
+# TECHDIR
+if [ ! -d "$TECHDIR" ]
+then
+ mkdir -p "$TECHDIR"
+ if [ "$?" == 0 ]
+ then
+ logit "INFO" "Folder $TECHDIR created!"
+ else
+ logit "ERROR" "Problem occured while creating $TECHDIR"
+ echo "ERROR Can't create $TECHDIR"
+ exit 1
+ fi
+else
+ logit "INFO" "Temp dir found"
+fi
+
+# FILEMAPSDIR
+if [ ! -d "$FILEMAPSDIR" ]
+then
+ mkdir -p "$FILEMAPSDIR"
+ if [ "$?" == 0 ]
+ then
+ logit "INFO" "Folder $FILEMAPSDIR created!"
+ else
+ logit "ERROR" "Problem occured while creating $FILEMAPSDIR"
+ echo "ERROR Can't create $FILEMAPSDIR"
+ exit 1
+ fi
+else
+ logit "INFO" "Mapsdir found"
+fi
+
+# CHKDIR
+if [ ! -d "$CHKDIR" ]
+then
+ mkdir -p "$CHKDIR"
+ if [ "$?" == 0 ]
+ then
+ logit "INFO" "Folder $CHKDIR created!"
+ else
+ logit "ERROR" "Problem occured while creating $CHKDIR"
+ echo "ERROR Can't create $CHKDIR"
+ exit 1
+ fi
+else
+ logit "INFO" "Workdir found"
+fi
+
+# MERKLEDIR
+if [ ! -d "$MERKLEDIR" ]
+then
+ mkdir -p "$MERKLEDIR"
+ if [ "$?" == 0 ]
+ then
+ logit "INFO" "Folder $MERKLEDIR created!"
+ else
+ logit "ERROR" "Problem occured while creating $MERKLEDIR"
+ echo "ERROR Can't create $MERKLEDIR"
+ exit 1
+ fi
+else
+ logit "INFO" "Workdir found"
+fi
+# Uncomment next line in case you want to debug the resulting script as well
+# echo 'set -xe' > $TEMPORARYDIR/cmd_queue.sh
+
+# We get the SHA512 hash for the $FILE given
+CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}')
+FILE_SIZE="$(du -b $FILE | awk '{ print $1 }')"
+if [ $FILE_SIZE -lt 4097 ]
+then
+ cp $FILE "$TECHDIR$(basename "$FILE")-00000000000000000000000000000000000000000000000000.chk"
+else
+ # We split the file into 4*1024 bytes and output the chunks into TECHDIR
+ split -a 50 -b 4096 --additional-suffix ".chk" -d "$FILE" "$TECHDIR$(basename "$FILE")-"
+fi
+
+# We go over there...
+cd $TECHDIR
+#set -xe
+# We get every chunks' SHA512 and we craft a script to rename the chunks and
+# move them to CHKDIR
+for file in $TEMPORARYDIR/chks/*
+do
+ sha512sum $file >> $TEMPORARYDIR/map
+done
+# Append last chk if not even number
+appendLastIfNotEven(){
+ if [ "$(expr $(wc -l "$1" | awk '{ print $1 }') % 2)" != 0 ]
+ then
+ tail -n 1 "$1" >> "$1"
+ fi
+}
+appendLastIfNotEven "$TEMPORARYDIR/map"
+# Figure out how many times we need to pack
+totalChunks=`grep 'chk' $TEMPORARYDIR/map | wc -l`
+temp="$totalChunks"
+timesRan=0
+while [ $temp != 1 ]
+do
+ temp=`expr $temp / 2`
+ timesRan=`expr $timesRan + 1`
+done
+printf "Ran %s times \n" "$timesRan"
+printf "Total chunks %s \n" "$totalChunks"
+
+workingIndex="$TEMPORARYDIR/map"
+c=$timesRan
+while [ "$c" != 0 ]
+do
+ a=1
+ printf "Level: %s, will work on %s chunks\n" "$c" "$totalChunks"
+ while [[ "$a" -lt "$totalChunks" ]]
+ do
+ b=`expr "$a" + 1`
+ sed -n "$a"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b
+ sed -n "$b"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b
+ shaSum="$(sha512sum level.$c.pair.$a-$b | awk '{ print $1 }')"
+ cp level.$c.pair.$a-$b $MERKLEDIR/$shaSum
+ sha512sum level.$c.pair.$a-$b | awk '{ print $1 }' >> level.$c.map
+ a=`expr "$a" + 2`
+ done
+ workingIndex="level.$c.map"
+ appendLastIfNotEven "$workingIndex"
+ shaSum=`sha512sum $workingIndex | awk '{ print $1 }'`
+ cp $workingIndex $MERKLEDIR/$shaSum
+ totalChunks=`cat $workingIndex | wc -l`
+ c=`expr $c - 1`
+done
+
+if [ -f level.1.map ]
+then
+ sha512sum level.1.map
+else
+ echo error
+ exit 1
+fi
+
+# Reset file with uniq
+cat $TEMPORARYDIR/map | uniq > $TEMPORARYDIR/map2
+cat $TEMPORARYDIR/map2 > $TEMPORARYDIR/map
+rm $TEMPORARYDIR/map2
+
+counter=0
+while IFS="" read -r p || [ -n "$p" ]
+do
+ printf "mv %s %s%s\n" "$(echo $p | awk '{ print $2 }')" "$CHKDIR" "$(echo $p | awk '{ print $1 }')" >> $TEMPORARYDIR/cmd_queue.sh
+ counter=`expr "$counter" + 1`
+done < $TEMPORARYDIR/map
+
+# We run the crafted script
+sh $TEMPORARYDIR/cmd_queue.sh
+
+# and we delete it
+rm $TEMPORARYDIR/cmd_queue.sh
+
+# We inform the map about the original $FILE name and SHA512
+echo "$CHECKSUM $(basename "$FILE")" >> $TEMPORARYDIR/map
+
+# We get the SHA512 hash of the resulted map file
+MAPFILEHASH="$(sha512sum $TEMPORARYDIR/map | awk '{ print $1 }')"
+
+# and we rename it with it and move it to FILEMAPSDIR
+`sha512sum $TEMPORARYDIR/map | awk '{print "mv " $2 " '$FILEMAPSDIR'" $1}'`
+
+# We remove the TEMPORARYDIR
+rm -rf $TEMPORARYDIR
+
+# and print the MAPFILEHASH
+echo "$MAPFILEHASH"