#!/bin/bash #set -x # # The concept is bit more complicated now # # 1. For a given file we split in 4KB files inside a temporary directory # # 2. We then create a map file and a merkle tree containing the resulted files # and their sha512sum. # # 3. We move the splitted files to our $CHKDIR named after their checksums # # 4. We move the merkle tree pairs to $MERKLEDIR # # We ultimately want to be seeding the file so # # 4. We append the checksum of the original file with its name into the map file # # 5. We rename the map file after its checksum and move it to maps directory # # 6. We are done! # # Uncomment next line if you want to debug # set -xe PROGRAM="$(basename $0)" if [ ! -f "$1" ] then echo "ERROR File not found" exit 1 else base64 $1 > file FILE="file" fi # The directory where the chunked data will be living at CHKDIR="$AK_WORKDIR/ftr" # The directory for the map files so we can reconstruct the file FILEMAPSDIR="$AK_WORKDIR/fmp" # Merkle tree file/references MERKLEDIR="$AK_WORKDIR/fmrk" # A temporary root dir to work on TEMPORARYDIR="$(ak-tempassin)" # A subdir to split the files there TECHDIR="$TEMPORARYDIR/chks" # A pin point to return from where we came from CURRENTDIR="$(pwd)" # Our snippet for logging debug info source $AK_LIBDIR/_ak_log # Checking directories and create them if necessary rm -rf $TEMPORARYDIR # TECHDIR if [ ! -d "$TECHDIR" ] then mkdir -p "$TECHDIR" if [ "$?" == 0 ] then logit "INFO" "Folder $TECHDIR created!" else logit "ERROR" "Problem occured while creating $TECHDIR" echo "ERROR Can't create $TECHDIR" exit 1 fi else logit "INFO" "Temp dir found" fi # FILEMAPSDIR if [ ! -d "$FILEMAPSDIR" ] then mkdir -p "$FILEMAPSDIR" if [ "$?" == 0 ] then logit "INFO" "Folder $FILEMAPSDIR created!" else logit "ERROR" "Problem occured while creating $FILEMAPSDIR" echo "ERROR Can't create $FILEMAPSDIR" exit 1 fi else logit "INFO" "Mapsdir found" fi # CHKDIR if [ ! -d "$CHKDIR" ] then mkdir -p "$CHKDIR" if [ "$?" == 0 ] then logit "INFO" "Folder $CHKDIR created!" else logit "ERROR" "Problem occured while creating $CHKDIR" echo "ERROR Can't create $CHKDIR" exit 1 fi else logit "INFO" "Workdir found" fi # MERKLEDIR if [ ! -d "$MERKLEDIR" ] then mkdir -p "$MERKLEDIR" if [ "$?" == 0 ] then logit "INFO" "Folder $MERKLEDIR created!" else logit "ERROR" "Problem occured while creating $MERKLEDIR" echo "ERROR Can't create $MERKLEDIR" exit 1 fi else logit "INFO" "Workdir found" fi # Uncomment next line in case you want to debug the resulting script as well # echo 'set -xe' > $TEMPORARYDIR/cmd_queue.sh # We get the SHA512 hash for the $FILE given CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}') FILE_SIZE="$(du -b $FILE | awk '{ print $1 }')" if [ $FILE_SIZE -lt 4097 ] then cp $FILE "$TECHDIR/$(basename "$FILE")-00000000000000000000000000000000000000000000000000.chk" else FACTOR=1024 while [ $(( $FILE_SIZE / $FACTOR )) -gt 250 ] do FACTOR=$(( $FACTOR * 2 )) done logit "INFO" "Gonna split in $FACTOR size" sleep 30 # We split the file into 4*1024 bytes and output the chunks into TECHDIR split -a 50 -b $FACTOR --additional-suffix ".chk" -d "$FILE" "$TECHDIR/$(basename "$FILE")-" fi # We go over there... cd $TECHDIR #set -xe # We get every chunks' SHA512 and we craft a script to rename the chunks and # move them to CHKDIR for file in $TEMPORARYDIR/chks/* do sha512sum $file >> $TEMPORARYDIR/map done # Append last chk if not even number appendLastIfNotEven(){ if [ "$(expr $(wc -l "$1" | awk '{ print $1 }') % 2)" != 0 ] then tail -n 1 "$1" >> "$1" fi } appendLastIfNotEven "$TEMPORARYDIR/map" # Figure out how many times we need to pack totalChunks=`grep 'chk' $TEMPORARYDIR/map | wc -l` temp="$totalChunks" timesRan=0 while [ $temp != 1 ] do temp=`expr $temp / 2` timesRan=`expr $timesRan + 1` done printf "Ran %s times \n" "$timesRan" printf "Total chunks %s \n" "$totalChunks" workingIndex="$TEMPORARYDIR/map" c=$timesRan while [ "$c" != 0 ] do a=1 printf "Level: %s, will work on %s chunks\n" "$c" "$totalChunks" while [[ "$a" -lt "$totalChunks" ]] do b=`expr "$a" + 1` sed -n "$a"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b sed -n "$b"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b shaSum="$(sha512sum level.$c.pair.$a-$b | awk '{ print $1 }')" cp level.$c.pair.$a-$b $MERKLEDIR/$shaSum sha512sum level.$c.pair.$a-$b | awk '{ print $1 }' >> level.$c.map a=`expr "$a" + 2` done workingIndex="level.$c.map" appendLastIfNotEven "$workingIndex" shaSum=`sha512sum $workingIndex | awk '{ print $1 }'` cp $workingIndex $MERKLEDIR/$shaSum totalChunks=`cat $workingIndex | wc -l` c=`expr $c - 1` done if [ -f level.1.map ] then sha512sum level.1.map else echo error exit 1 fi # Reset file with uniq cat $TEMPORARYDIR/map | uniq > $TEMPORARYDIR/map2 cat $TEMPORARYDIR/map2 > $TEMPORARYDIR/map rm $TEMPORARYDIR/map2 counter=0 while IFS="" read -r p || [ -n "$p" ] do printf "mv %s %s%s\n" "$(echo $p | awk '{ print $2 }')" "$CHKDIR" "$(echo $p | awk '{ print $1 }')" >> $TEMPORARYDIR/cmd_queue.sh counter=`expr "$counter" + 1` done < $TEMPORARYDIR/map # We run the crafted script sh $TEMPORARYDIR/cmd_queue.sh # and we delete it rm $TEMPORARYDIR/cmd_queue.sh # We inform the map about the original $FILE name and SHA512 echo "$CHECKSUM $(basename "$FILE")" >> $TEMPORARYDIR/map # We get the SHA512 hash of the resulted map file MAPFILEHASH="$(sha512sum $TEMPORARYDIR/map | awk '{ print $1 }')" # and we rename it with it and move it to FILEMAPSDIR `sha512sum $TEMPORARYDIR/map | awk '{print "mv " $2 " '$FILEMAPSDIR'" $1}'` # We remove the TEMPORARYDIR rm -rf $TEMPORARYDIR # and print the MAPFILEHASH echo "$MAPFILEHASH"