diff options
Diffstat (limited to 'bin/ak-fs-add')
| -rwxr-xr-x | bin/ak-fs-add | 226 |
1 files changed, 226 insertions, 0 deletions
diff --git a/bin/ak-fs-add b/bin/ak-fs-add new file mode 100755 index 0000000..2ba5e91 --- /dev/null +++ b/bin/ak-fs-add @@ -0,0 +1,226 @@ +#!/bin/bash +# +# The concept is bit more complicated now +# +# 1. For a given file we split in 4KB files inside a temporary directory +# +# 2. We then create a map file and a merkle tree containing the resulted files +# and their sha512sum. +# +# 3. We move the splitted files to our $CHKDIR named after their checksums +# +# 4. We move the merkle tree pairs to $MERKLEDIR +# +# We ultimately want to be seeding the file so +# +# 4. We append the checksum of the original file with its name into the map file +# +# 5. We rename the map file after its checksum and move it to maps directory +# +# 6. We are done! +# + +# Uncomment next line if you want to debug +# set -xe +PROGRAM="$(basename $0)" + +if [ ! -f "$1" ] +then + echo "ERROR File not found" + exit 1 +else + base64 $1 > file + FILE="file" +fi + +# The directory where the chunked data will be living at +CHKDIR="$AK_WORKDIR/ftr/" +# The directory for the map files so we can reconstruct the file +FILEMAPSDIR="$AK_WORKDIR/fmp/" +# Merkle tree file/references +MERKLEDIR="$AK_WORKDIR/fmrk/" +# A temporary root dir to work on +TEMPORARYDIR="$(ak-tempassin)" +# A subdir to split the files there +TECHDIR="$TEMPORARYDIR/chks/" +# A pin point to return from where we came from +CURRENTDIR="$(pwd)/" + +# Our snippet for logging debug info +logit(){ + ak-logthis "$PROGRAM" "$1" "$2" +} + +# Checking directories and create them if necessary +rm -rf $TEMPORARYDIR + +# TECHDIR +if [ ! -d "$TECHDIR" ] +then + mkdir -p "$TECHDIR" + if [ "$?" == 0 ] + then + logit "INFO" "Folder $TECHDIR created!" + else + logit "ERROR" "Problem occured while creating $TECHDIR" + echo "ERROR Can't create $TECHDIR" + exit 1 + fi +else + logit "INFO" "Temp dir found" +fi + +# FILEMAPSDIR +if [ ! -d "$FILEMAPSDIR" ] +then + mkdir -p "$FILEMAPSDIR" + if [ "$?" == 0 ] + then + logit "INFO" "Folder $FILEMAPSDIR created!" + else + logit "ERROR" "Problem occured while creating $FILEMAPSDIR" + echo "ERROR Can't create $FILEMAPSDIR" + exit 1 + fi +else + logit "INFO" "Mapsdir found" +fi + +# CHKDIR +if [ ! -d "$CHKDIR" ] +then + mkdir -p "$CHKDIR" + if [ "$?" == 0 ] + then + logit "INFO" "Folder $CHKDIR created!" + else + logit "ERROR" "Problem occured while creating $CHKDIR" + echo "ERROR Can't create $CHKDIR" + exit 1 + fi +else + logit "INFO" "Workdir found" +fi + +# MERKLEDIR +if [ ! -d "$MERKLEDIR" ] +then + mkdir -p "$MERKLEDIR" + if [ "$?" == 0 ] + then + logit "INFO" "Folder $MERKLEDIR created!" + else + logit "ERROR" "Problem occured while creating $MERKLEDIR" + echo "ERROR Can't create $MERKLEDIR" + exit 1 + fi +else + logit "INFO" "Workdir found" +fi +# Uncomment next line in case you want to debug the resulting script as well +# echo 'set -xe' > $TEMPORARYDIR/cmd_queue.sh + +# We get the SHA512 hash for the $FILE given +CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}') +FILE_SIZE="$(du -b $FILE | awk '{ print $1 }')" +if [ $FILE_SIZE -lt 4097 ] +then + cp $FILE "$TECHDIR$(basename "$FILE")-00000000000000000000000000000000000000000000000000.chk" +else + # We split the file into 4*1024 bytes and output the chunks into TECHDIR + split -a 50 -b 4096 --additional-suffix ".chk" -d "$FILE" "$TECHDIR$(basename "$FILE")-" +fi + +# We go over there... +cd $TECHDIR +#set -xe +# We get every chunks' SHA512 and we craft a script to rename the chunks and +# move them to CHKDIR +for file in $TEMPORARYDIR/chks/* +do + sha512sum $file >> $TEMPORARYDIR/map +done +# Append last chk if not even number +appendLastIfNotEven(){ + if [ "$(expr $(wc -l "$1" | awk '{ print $1 }') % 2)" != 0 ] + then + tail -n 1 "$1" >> "$1" + fi +} +appendLastIfNotEven "$TEMPORARYDIR/map" +# Figure out how many times we need to pack +totalChunks=`grep 'chk' $TEMPORARYDIR/map | wc -l` +temp="$totalChunks" +timesRan=0 +while [ $temp != 1 ] +do + temp=`expr $temp / 2` + timesRan=`expr $timesRan + 1` +done +printf "Ran %s times \n" "$timesRan" +printf "Total chunks %s \n" "$totalChunks" + +workingIndex="$TEMPORARYDIR/map" +c=$timesRan +while [ "$c" != 0 ] +do + a=1 + printf "Level: %s, will work on %s chunks\n" "$c" "$totalChunks" + while [[ "$a" -lt "$totalChunks" ]] + do + b=`expr "$a" + 1` + sed -n "$a"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b + sed -n "$b"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b + shaSum="$(sha512sum level.$c.pair.$a-$b | awk '{ print $1 }')" + cp level.$c.pair.$a-$b $MERKLEDIR/$shaSum + sha512sum level.$c.pair.$a-$b | awk '{ print $1 }' >> level.$c.map + a=`expr "$a" + 2` + done + workingIndex="level.$c.map" + appendLastIfNotEven "$workingIndex" + shaSum=`sha512sum $workingIndex | awk '{ print $1 }'` + cp $workingIndex $MERKLEDIR/$shaSum + totalChunks=`cat $workingIndex | wc -l` + c=`expr $c - 1` +done + +if [ -f level.1.map ] +then + sha512sum level.1.map +else + echo error + exit 1 +fi + +# Reset file with uniq +cat $TEMPORARYDIR/map | uniq > $TEMPORARYDIR/map2 +cat $TEMPORARYDIR/map2 > $TEMPORARYDIR/map +rm $TEMPORARYDIR/map2 + +counter=0 +while IFS="" read -r p || [ -n "$p" ] +do + printf "mv %s %s%s\n" "$(echo $p | awk '{ print $2 }')" "$CHKDIR" "$(echo $p | awk '{ print $1 }')" >> $TEMPORARYDIR/cmd_queue.sh + counter=`expr "$counter" + 1` +done < $TEMPORARYDIR/map + +# We run the crafted script +sh $TEMPORARYDIR/cmd_queue.sh + +# and we delete it +rm $TEMPORARYDIR/cmd_queue.sh + +# We inform the map about the original $FILE name and SHA512 +echo "$CHECKSUM $(basename "$FILE")" >> $TEMPORARYDIR/map + +# We get the SHA512 hash of the resulted map file +MAPFILEHASH="$(sha512sum $TEMPORARYDIR/map | awk '{ print $1 }')" + +# and we rename it with it and move it to FILEMAPSDIR +`sha512sum $TEMPORARYDIR/map | awk '{print "mv " $2 " '$FILEMAPSDIR'" $1}'` + +# We remove the TEMPORARYDIR +rm -rf $TEMPORARYDIR + +# and print the MAPFILEHASH +echo "$MAPFILEHASH" |