#!/usr/bin/env bash source $AK_LIBDIR/_ak_log source $AK_LIBDIR/_ak_hash_exchange function _ak_fs_dir_init_setup(){ _ak_check_and_create_dir $AK_MAPSDIR _ak_check_and_create_dir $AK_CHUNKSDIR _ak_check_and_create_dir $AK_LEAFSDIR } function _ak_fs_return_hash_path(){ #hashpath="$(echo -n "$1" |sed 's/./&\//g;s/\/$//g')" #hashpath="$(echo -n "$1" | fold -w 4 | tr $'\n' '/')" echo -n "$1" } function _ak_fs_return_hash_dir(){ #hashdir="$(_ak_fs_return_hash_path $1 | sed -e 's/.....$//')" #echo -n "$hashdir" echo -n "$1" } function _ak_fs_verify_input_is_hash(){ if [ ! -z "$1" ] && echo "$1" | grep '[0123456789abcdef]\{128\}' > /dev/null 2>&1 then return 0 else _ak_log_error "no hash?!" exit 1 fi } function _ak_fs_create_dir_for_hash(){ _ak_fs_verify_input_is_hash $2 if [ ! -z $1 ] && [ ! -z $2 ] && [ -n "$1" ] && [ -n "$2" ] then mkdir -p "$1/$(_ak_fs_return_hash_dir $2)" fi } # Append last chk if not even number function _ak_fs_appendLastIfNotEven(){ if [ "$(( $(wc -l "$1" | awk '{ print $1 }') % 2))" -ne 0 ] then tail -n 1 "$1" >> "$1.temp" cat "$1.temp" >> "$1" rm $1.temp fi } function _ak_fs_import(){ # # The concept is bit more complicated now # # 1. For a given file we split in 4KB files inside a temporary directory # # 2. We then create a map file and a merkle tree containing the resulted files # and their sha512sum. # # 3. We move the splitted files to our $AK_CHUNKSDIR named after their checksums # # 4. We move the merkle tree pairs to $AK_LEAFSDIR # # We ultimately want to be seeding the file so # # 4. We append the checksum of the original file with its name into the map file # # 5. We rename the map file after its checksum and move it to maps directory # # 6. We are done! # # A temporary root dir to work on TEMPDIR="$(_ak_make_temp_directory)" # A subdir to split the files there TCHKDIR="$TEMPDIR/chks" # A pin point to return from where we came from CURRENTDIR="$(pwd)" # Checking directories and create them if necessary # rm -rf $TEMPDIR _ak_check_and_create_dir $TCHKDIR _ak_fs_dir_init_setup if [ ! -f "$1" ] then _ak_log_error "File $1 not found" exit 1 else # TODO # Side hustle, save the original hash and original filename # This won't be expected in general # The idea is: # sha256sum as quick pointer to root of the tree # _ak_log_info "Storing original hash of $1 along with its name" sha512sum "$1" > $TEMPDIR/3rd_gen_map _ak_log_info "Encoding to base64" base64 $1 > file FILE="file" fi # Uncomment next line in case you want to debug the resulting script as well # echo 'set -xe' > $TEMPDIR/cmd_queue.sh # We get the SHA512 hash for the $FILE given CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}') FILE_SIZE="$( (du -b $FILE||stat -f %z $FILE)2>/dev/null | awk '{ print $1 }')" if [ $FILE_SIZE -lt 4097 ] then cp $FILE "$TCHKDIR/chunk-0000000000000" else FACTOR=1024 while [ $(( $FILE_SIZE / $FACTOR )) -gt 250 ] do FACTOR=$(( $FACTOR * 2 )) done _ak_log_info "Gonna split in $FACTOR size" sleep 3 # We split the file into 4*1024 bytes and output the chunks into TCHKDIR split -a 13 -b $FACTOR -d "$FILE" "$TCHKDIR/chunk-" fi _ak_log_info "File done splitting" # We go over there... cd $TCHKDIR #set -xe # We get every chunks' SHA512 and we craft a script to rename the chunks and # move them to AK_CHUNKSDIR for file in * do _ak_log_debug "Hashing ${file}..." sha512sum $file >> $TEMPDIR/map _ak_log_debug "Hashing of ${file} completed" done _ak_fs_appendLastIfNotEven "$TEMPDIR/map" # Figure out how many times we need to pack totalChunks=`grep 'chunk-' $TEMPDIR/map | wc -l` temp="$totalChunks" timesRan=0 while [ $temp -ne 1 ] do temp=`expr $temp / 2` timesRan=`expr $timesRan + 1` done _ak_log_debug "Ran $timesRan times" _ak_log_debug "Total chunks $totalChunks" workingIndex="$TEMPDIR/map" c=$timesRan while [ $c -ne 0 ] do a=1 _ak_log_debug "Level: $c, will work on $totalChunks chunks" while [[ "$a" -lt "$totalChunks" ]] do b=`expr "$a" + 1` sed -n "$a"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b sed -n "$b"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b shaSum="$(sha512sum level.$c.pair.$a-$b | awk '{ print $1 }')" #mkdir -p $AK_LEAFSDIR/$(_ak_fs_return_hash_dir $shaSum) #cp level.$c.pair.$a-$b $AK_LEAFSDIR/$(_ak_fs_return_hash_path $shaSum) cp level.$c.pair.$a-$b $AK_LEAFSDIR/$shaSum sha512sum level.$c.pair.$a-$b | awk '{ print $1 }' >> level.$c.map a=`expr "$a" + 2` done workingIndex="level.$c.map" _ak_fs_appendLastIfNotEven "$workingIndex" shaSum=`sha512sum $workingIndex | awk '{ print $1 }'` #mkdir -p $AK_LEAFSDIR/$(_ak_fs_return_hash_dir $shaSum) #cp $workingIndex $AK_LEAFSDIR/$(_ak_fs_return_hash_path $shaSum) cp $workingIndex $AK_LEAFSDIR/$shaSum # cp $workingIndex $AK_LEAFSDIR/$shaSum totalChunks=`cat $workingIndex | wc -l` c=`expr $c - 1` done if [ -f level.1.map ] then sha512sum level.1.map sha512sum level.1.map >> $TEMPDIR/3rd_gen_map else echo error exit 1 fi # Reset file with uniq cat $TEMPDIR/map | uniq > $TEMPDIR/map2 cat $TEMPDIR/map2 > $TEMPDIR/map rm $TEMPDIR/map2 counter=0 while IFS="" read -r p || [ -n "$p" ] do # printf "mv %s %s/%s\n" "$(echo $p | awk '{ print $2 }')" "$AK_CHUNKSDIR" "$(echo $p | awk '{ print $1 }')" >> $TEMPDIR/cmd_queue.sh #mkdir -p $AK_CHUNKSDIR/$(echo $p | awk '{ print $1 }') cp $(echo $p | awk '{ print $2 }') $AK_CHUNKSDIR/$(echo $p | awk '{ print $1 }') counter=`expr "$counter" + 1` done < $TEMPDIR/map # We run the crafted script # sh $TEMPDIR/cmd_queue.sh # and we delete it # rm $TEMPDIR/cmd_queue.sh # We inform the map about the original $FILE name and SHA512 # echo "$CHECKSUM $(basename "$FILE")" >> $TEMPDIR/map # We get the SHA512 hash of the resulted map file # MAPFILEHASH="$(sha512sum $TEMPDIR/map | awk '{ print $1 }')" # and we rename it with it and move it to AK_MAPSDIR # `sha512sum $TEMPDIR/map | awk '{print "mv " $2 " '$AK_MAPSDIR/'" $1}'` mp512p="$(sha512sum $TEMPDIR/3rd_gen_map | awk '{print $1}')" cp $TEMPDIR/3rd_gen_map $AK_MAPSDIR/$mp512p # We remove the TEMPDIR rm -rf $TEMPDIR # and print the MAPFILEHASH echo "$mp512p" } function _ak_fs_find_depth(){ currentNode="$1" #pathToNode="$AK_LEAFSDIR/$(_ak_fs_return_hash_path $currentNode)" pathToNode="$AK_LEAFSDIR/$currentNode" if [ -f $pathToNode ] && [ "$( (du -b $pathToNode||stat -f %z $pathToNode)2>/dev/null | awk '{print $1}')" == "258" ] then fileHead="$(head -n 1 $pathToNode)" counter="$(expr $counter + 1)" _ak_fs_find_depth "$fileHead" elif [ ! -f $pathToNode ] then printf "%s" "$counter" > depth else exit 111 # Try to download stuff # curl -s $remoteMrk/$currentNode -o $AK_LEAFSDIR/$currentNode # if [ $? -ne 0 ] # then # exit 111 # fi # _ak_fs_find_depth "$currentNode" fi } function _ak_fs_cat(){ if [ -z $1 ] then echo "Please provide a SHA512 hash" exit 1 fi echo $1 | grep "[0123456789abcdef]\{128\}" if [ $? -ne 0 ] then echo "Look, I asked for a SHA512 hash, please try again" exit 1 fi treeRootHash="$1" # Enter temp folder TEMPDIR="$(_ak_make_temp_directory)" cd $TEMPDIR currentNode="$treeRootHash" counter=0 printf "%s" "$currentNode" > workspace.0 _ak_fs_find_depth "$currentNode" depth="$(expr `cat depth` + 1)" counter="0" printf "%s" "$depth" if [ -f output ] then rm output fi touch output while [ "$counter" != "$depth" ] do _ak_log_debug "Entering loop... $counter $depth" while IFS="" read -r p || [ -n "$p" ] do nextLevel="$(expr $counter + 1)" if [ "$p" == "" ] then echo hi else #expectedPath="$AK_LEAFSDIR/$(_ak_fs_return_hash_path $p)" expectedPath="$AK_LEAFSDIR/$p" if [ -f $expectedPath ] then if [ "$(head -n 1 $expectedPath)" == "$(tail -n 1 $expectedPath)" ] then head -n 1 $expectedPath >> workspace.$nextLevel else cat $expectedPath >> workspace.$nextLevel fi #elif [ -f $AK_CHUNKSDIR/$(_ak_fs_return_hash_path $p) ] elif [ -f $AK_CHUNKSDIR/$p ] then #cat $AK_CHUNKSDIR/$(_ak_fs_return_hash_path $p) >> output cat $AK_CHUNKSDIR/$p >> output fi fi done < workspace.$counter counter="$(expr $counter + 1)" done base64 -d output _ak_log_info "Recreation of $treeRootHash succeeded!" } function _ak_fs_export(){ if [ -z $1 ] then _ak_log_error "Please provide a SHA512 hash" exit 1 fi if [ -z $2 ] then _ak_log_error "Please an output filename" exit 2 fi outputFilename="$2" echo $1 | grep "[0123456789abcdef]\{128\}" if [ $? -ne 0 ] then _ak_log_error "Look, I asked for a SHA512 hash, please try again" exit 1 fi _ak_fs_export "$1" > $outputFilename } function _ak_fs_list(){ if [ -d "${AK_MAPSDIR}" ] then find $AK_MAPSDIR -type f | while read fina do cat $fina | tr '\n' ' ' | awk '{ print "'$(basename ${fina})' " $3 " " $2 }' done else _ak_log_debug "Making ${AK_MAPSDIR} directory" mkdir -p ${AK_MAPSDIR} _ak_log_debug "Empty directory" fi } function _ak_fs_from_map_get_original_hash(){ if [ ! -z "$1" ] && [ -n "$1" ] then if [ -f "$AK_MAPSDIR/$1" ] then cat $AK_MAPSDIR/$1 | tr '\n' ' ' | awk '{ print $1 }' else _ak_log_error "Map $1 was not found" fi else _ak_log_error "No input given" fi } function _ak_fs_from_map_get_original_filename(){ if [ ! -z "$1" ] && [ -n "$1" ] then if [ -f "$AK_MAPSDIR/$1" ] then cat $AK_MAPSDIR/$1 | tr '\n' ' ' | awk '{ print $2 }' else _ak_log_error "Map $1 was not found" fi else _ak_log_error "No input given" fi } function _ak_fs_from_map_get_root_hash(){ if [ ! -z "$1" ] && [ -n "$1" ] then if [ -f "$AK_MAPSDIR/$1" ] then cat $AK_MAPSDIR/$1 | tr '\n' ' ' | awk '{ print $3 }' else _ak_log_error "Map $1 was not found" fi else _ak_log_error "No input given" fi } function _ak_fs_cat_from_map_hash(){ if _ak_fs_verify_input_is_hash "$1" then _ak_fs_cat `_ak_fs_from_map_get_root_hash $1` fi } function _ak_fs_get_from_map_hash(){ if _ak_fs_verify_input_is_hash "$1" then _ak_fs_get `_ak_fs_from_map_get_root_hash $1` "`_ak_fs_from_map_get_original_filename $1`" fi } function _ak_net_try_leaf(){ if [ ! -z "$1" ] && [ -n "$1" ] then if [ ! -f "$AK_LEAFSDIR/$1" ] then _ak_he_list_peers | while read peer do _ak_log_info "Searching as a leaf... $1 @ ${peer}" request_url="$(_ak_he_url_request_leaf_hash_from_peer ${peer} $1)" curl -s --connect-timeout 3 -o $1 ${request_url} if [ $? -ne 0 ] then _ak_log_warning "Curl to ${request_url} failed." continue fi if [ -f $1 ] then _ak_log_info "${request_url} gave out a file" if [ "$(sha512sum $1 | awk '{print $1}')" == "$1" ] then _ak_log_info "File hash match" if [ "$(cat $1 | wc -l)" == 2 ] then _ak_log_info "File has two lines" if [ "$(cat $1 | grep '^[0-9a-z]\{128\}$' | wc -l)" == 2 ] then _ak_log_info "File has two lines which are 2 hashes" cat $1 cp $1 ${AK_LEAFSDIR}/$1 exit break else _ak_log_warning "File $1 doesn't match expected format (two hashes)" rm $1 fi else _ak_log_warning "File $1 doesn't match expected format (two lines)" rm $1 fi else cat $1 | jq >&2 _ak_log_warning "File $1 doesn't match expected hash: $(sha512sum $1 | awk '{print $1}')" rm $1 fi else _ak_log_warning "No file $1" fi done fi fi } function _ak_net_try_chunk(){ if [ ! -z "$1" ] && [ -n "$1" ] then if [ ! -f "$AK_CHUNKSDIR/$1" ] then _ak_he_list_peers | while read peer do _ak_log_info "Searching as a chunk... $1 @ ${peer}" request_url="$(_ak_he_url_request_chunk_hash_from_peer ${peer} $1)" curl -s --connect-timeout 3 -o $1 ${request_url} if [ $? -ne 0 ] then _ak_log_warning "Curl to $request_url failed." continue fi if [ -f $1 ] then _ak_log_info "File found" if [ "$(sha512sum $1 | awk '{print $1}')" == "$1" ] then _ak_log_info "File is valid" if [ "$(cat $1 | grep -v '^[-A-Za-z0-9+/]*=\{0,3\}$')" == "" ] then _ak_log_info "File is base64" cat $1 cp $1 ${AK_CHUNKSDIR}/$1 exit break else _ak_log_warning "File $1 doesn't match expected format (base64)" rm $1 fi else cat $1 | jq >&2 _ak_log_warning "File $1 doesn't match expected hash: $(sha512sum $1 | awk '{print $1}')" rm $1 fi else _ak_log_warning "No file $1" fi done fi fi } function _ak_fs_net_find_depth(){ currentNode="$1" pathToNode="$AK_LEAFSDIR/$currentNode" _ak_net_try_leaf ${currentNode} _ak_net_try_chunk ${currentNode} if [ -f $pathToNode ] && [ "$( (du -b $pathToNode||stat -f %z $pathToNode)2>/dev/null | awk '{print $1}')" == "258" ] then fileHead="$(head -n 1 $pathToNode)" counter="$(expr $counter + 1)" _ak_fs_net_find_depth "$fileHead" elif [ ! -f $pathToNode ] then printf "%s" "$counter" > depth else pwd exit 111 # Try to download stuff # curl -s $remoteMrk/$currentNode -o $AK_LEAFSDIR/$currentNode # if [ $? -ne 0 ] # then # exit 111 # fi # _ak_fs_find_depth "$currentNode" fi } function _ak_fs_net_cat(){ if [ -z $1 ] then echo "Please provide a SHA512 hash" exit 1 fi echo $1 | grep "[0123456789abcdef]\{128\}" if [ $? -ne 0 ] then echo "Look, I asked for a SHA512 hash, please try again" exit 1 fi treeRootHash="$1" TEMPDIR="$(_ak_make_temp_directory)" cd $TEMPDIR _ak_log_debug "We are here: $(pwd)" currentNode="$treeRootHash" counter=0 printf "%s" "$currentNode" > workspace.0 while [ ! -f "$currentNode" ] do curl -s -o ${currentNode} $(_ak_he_url_request_leaf_hash_from_random_peer ${currentNode}) done _ak_fs_net_find_depth "$currentNode" depth="$(expr `cat depth` + 1)" counter="0" printf "%s" "$depth" if [ -f output ] then rm output fi touch output while [ "$counter" -lt "$depth" ] do _ak_log_debug "Entering loop... $counter $depth" while IFS="" read -r p || [ -n "$p" ] do nextLevel="$(expr $counter + 1)" if [ "$p" == "" ] then echo hi else expectedPath="$AK_LEAFSDIR/$p" if [ -f $expectedPath ] then _ak_log_info "Leaf available" if [ "$(head -n 1 $expectedPath)" == "$(tail -n 1 $expectedPath)" ] then head -n 1 $expectedPath >> workspace.$nextLevel else cat $expectedPath >> workspace.$nextLevel fi elif [ -f $AK_CHUNKSDIR/$p ] then _ak_log_info "Chunk available" cat $AK_CHUNKSDIR/$p >> output else _ak_log_info "Retrying with network" _ak_net_try_leaf $p _ak_net_try_chunk $p _ak_log_info "After network try..." if [ -f $AK_LEAFSDIR/$1 ] then _ak_log_info "Leaf available over net" if [ "$(head -n 1 $AK_LEAFSDIR/$1)" == "$(tail -n 1 $AK_LEAFSDIR/$1)" ] then head -n 1 $AK_LEAFSDIR/$1 >> workspace.$nextLevel else cat $AK_LEAFSDIR/$1 >> workspace.$nextLevel fi elif [ -f $AK_CHUNKSDIR/$p ] then _ak_log_info "Chunk available over net" cat $AK_CHUNKSDIR/$p >> output fi fi fi done < workspace.$counter counter="$(expr $counter + 1)" done base64 -d output > output.d shaout="$(sha512sum output.d | awk '{print $1}')" expHash="$2" if [ "$shaout" == "$expHash" ] then base64 -d output _ak_log_info "Recreation of $treeRootHash succeeded!" else _ak_log_error "Recreation of $treeRootHash failed!" _ak_log_error "Expected: $expHash" _ak_log_error "Got: $shaout" fi } function _ak_fs_from_map_net_get_original_hash(){ TEMPDIR=$(_ak_make_temp_directory) cd $TEMPDIR if _ak_fs_verify_input_is_hash "$1" then _ak_he_list_peers | while read peer do _ak_log_debug "Trying $(_ak_he_url_request_map_hash_from_peer ${peer} $1)..." curl -s --connect-timeout 3 -o ${TEMPDIR}/peer_${peer}.reply $(_ak_he_url_request_map_hash_from_peer ${peer} $1) done find . -type f | while read reply do cat ${reply} | jq >/dev/null 2>&1 if [ $? -eq 0 ] then _ak_log_error "Found error in reply: $(cat ${reply} | jq -r '.error')" else if [ $(cat ${reply} | wc -l) -ne 2 ] then _ak_log_error "${reply} is not two lines long" exit 1 else _ak_log_info "${reply} is two lines long" fi if [ $(cat ${reply} | grep '^[0-9a-z]\{128\} ' | wc -l) -eq 2 ] then _ak_log_info "${reply} contains two hashes" else _ak_log_error "${reply} doesn't contain the appropriate info" exit 1 fi cat ${reply} | head -n 1 | awk '{print $1}' break fi done fi } function _ak_fs_from_map_net_get_root_hash(){ TEMPDIR=$(_ak_make_temp_directory) cd $TEMPDIR if _ak_fs_verify_input_is_hash "$1" then _ak_he_list_peers | while read peer do _ak_log_debug "Trying $(_ak_he_url_request_map_hash_from_peer ${peer} $1)..." curl -s --connect-timeout 3 -o ${TEMPDIR}/peer_${peer}.reply $(_ak_he_url_request_map_hash_from_peer ${peer} $1) done find . -type f | while read reply do cat ${reply} | jq >/dev/null 2>&1 if [ $? -eq 0 ] then _ak_log_error "Found error in reply: $(cat ${reply} | jq -r '.error')" else if [ $(cat ${reply} | wc -l) -ne 2 ] then _ak_log_error "${reply} is not two lines long" exit 1 else _ak_log_info "${reply} is two lines long" fi if [ $(cat ${reply} | grep '^[0-9a-z]\{128\} ' | wc -l) -eq 2 ] then _ak_log_info "${reply} contains two hashes" else _ak_log_error "${reply} doesn't contain the appropriate info" exit 1 fi cat ${reply} | grep 'level.1.map' | awk '{print $1}' break fi done fi } function _ak_fs_net_cat_from_map_hash(){ if _ak_fs_verify_input_is_hash "$1" then _ak_fs_net_cat `_ak_fs_from_map_net_get_root_hash $1` `_ak_fs_from_map_net_get_original_hash $1` fi }