From 798a8a50445f894f7e6979b9e688639005d0a5d9 Mon Sep 17 00:00:00 2001 From: kaotisk Date: Sun, 10 Mar 2024 02:08:24 +0200 Subject: Renamed ak-sm-merkle-tree* to ak-fs-* renamed: bin/ak-sm-merkle-tree -> bin/ak-fs-add renamed: bin/ak-sm-merkle-tree-to-file -> bin/ak-fs-get Also, minor fixes in README --- README | 15 +-- bin/ak-fs-add | 226 ++++++++++++++++++++++++++++++++++++++++++ bin/ak-fs-get | 104 +++++++++++++++++++ bin/ak-sm-merkle-tree | 226 ------------------------------------------ bin/ak-sm-merkle-tree-to-file | 104 ------------------- 5 files changed, 339 insertions(+), 336 deletions(-) create mode 100755 bin/ak-fs-add create mode 100755 bin/ak-fs-get delete mode 100755 bin/ak-sm-merkle-tree delete mode 100755 bin/ak-sm-merkle-tree-to-file diff --git a/README b/README index c3172c0..feb2206 100644 --- a/README +++ b/README @@ -49,7 +49,7 @@ with `akconfig` a pointer to some basic information that you can share with others in order for them to crawl your zchain or view your "profile". There are more tools available under the `bin` folder. Make sure you deeply -understand what they are doing before using them. +understand what they are doing before using them. Changes are mentioned in `git log` of the repo, as well as in CHANGELOG file. @@ -125,7 +125,7 @@ signature of this, timestamp and finally an entry for the previous *zblock*. After that, we finally write this as a JSON object, add it to IPFS, sign it and pack a *ZBLOCK*. The ZBLOCK is then published over our IPNS zchain key. -Other options... let's try help! +Other options... let's try help! ``` console $ ak-news help @@ -144,7 +144,7 @@ create Vim is going to pop up, you will write and save your Clearly there is a TODO item. Import is not working so avoid it, or fix it. Add is nice, you can add an already existing file directly. `news` is the second module after `mixtape`. Both modules need refactoring but they work at a level -that someone can be productive with these tools. +that someone can be productive with these tools. ### Explore chains ( `ak-enter` ) @@ -247,7 +247,7 @@ ak-news # Adds a news article to the ZCHAIN ak-profile # Adds key-value pairs to the ZCHAIN ak-reference # Adds references to the ZCHAIN ak-repositories # Adds repositories to the ZCHAIN -ak-roadmap PROTO Adds a roadmap to the ZCHAIN +ak-roadmap PROTO Adds a roadmap to the ZCHAIN ak-transactions # Prototype of transactions in the ZCHAIN ak-sm-files # Adds a split file's map to the ZCHAIN ak-todos PROTO Adds todo list to the ZCHAIN @@ -276,8 +276,11 @@ File splitters -------------- ak-sm-filejoiner # Joins a file from a map ak-sm-filesplitter # Splits a file to chunks and creates a map -ak-sm-merkle-tree # Splits a file to merkle tree and returns the root -ak-sm-merkle-tree-to-file # Joins a file from a merkle root + +File system +----------- +ak-fs-add # Adds a file to filesystem tree (returns root hash) +ak-fs-get # Constructs a file from root hash IPFS Wrappers ------------- diff --git a/bin/ak-fs-add b/bin/ak-fs-add new file mode 100755 index 0000000..2ba5e91 --- /dev/null +++ b/bin/ak-fs-add @@ -0,0 +1,226 @@ +#!/bin/bash +# +# The concept is bit more complicated now +# +# 1. For a given file we split in 4KB files inside a temporary directory +# +# 2. We then create a map file and a merkle tree containing the resulted files +# and their sha512sum. +# +# 3. We move the splitted files to our $CHKDIR named after their checksums +# +# 4. We move the merkle tree pairs to $MERKLEDIR +# +# We ultimately want to be seeding the file so +# +# 4. We append the checksum of the original file with its name into the map file +# +# 5. We rename the map file after its checksum and move it to maps directory +# +# 6. We are done! +# + +# Uncomment next line if you want to debug +# set -xe +PROGRAM="$(basename $0)" + +if [ ! -f "$1" ] +then + echo "ERROR File not found" + exit 1 +else + base64 $1 > file + FILE="file" +fi + +# The directory where the chunked data will be living at +CHKDIR="$AK_WORKDIR/ftr/" +# The directory for the map files so we can reconstruct the file +FILEMAPSDIR="$AK_WORKDIR/fmp/" +# Merkle tree file/references +MERKLEDIR="$AK_WORKDIR/fmrk/" +# A temporary root dir to work on +TEMPORARYDIR="$(ak-tempassin)" +# A subdir to split the files there +TECHDIR="$TEMPORARYDIR/chks/" +# A pin point to return from where we came from +CURRENTDIR="$(pwd)/" + +# Our snippet for logging debug info +logit(){ + ak-logthis "$PROGRAM" "$1" "$2" +} + +# Checking directories and create them if necessary +rm -rf $TEMPORARYDIR + +# TECHDIR +if [ ! -d "$TECHDIR" ] +then + mkdir -p "$TECHDIR" + if [ "$?" == 0 ] + then + logit "INFO" "Folder $TECHDIR created!" + else + logit "ERROR" "Problem occured while creating $TECHDIR" + echo "ERROR Can't create $TECHDIR" + exit 1 + fi +else + logit "INFO" "Temp dir found" +fi + +# FILEMAPSDIR +if [ ! -d "$FILEMAPSDIR" ] +then + mkdir -p "$FILEMAPSDIR" + if [ "$?" == 0 ] + then + logit "INFO" "Folder $FILEMAPSDIR created!" + else + logit "ERROR" "Problem occured while creating $FILEMAPSDIR" + echo "ERROR Can't create $FILEMAPSDIR" + exit 1 + fi +else + logit "INFO" "Mapsdir found" +fi + +# CHKDIR +if [ ! -d "$CHKDIR" ] +then + mkdir -p "$CHKDIR" + if [ "$?" == 0 ] + then + logit "INFO" "Folder $CHKDIR created!" + else + logit "ERROR" "Problem occured while creating $CHKDIR" + echo "ERROR Can't create $CHKDIR" + exit 1 + fi +else + logit "INFO" "Workdir found" +fi + +# MERKLEDIR +if [ ! -d "$MERKLEDIR" ] +then + mkdir -p "$MERKLEDIR" + if [ "$?" == 0 ] + then + logit "INFO" "Folder $MERKLEDIR created!" + else + logit "ERROR" "Problem occured while creating $MERKLEDIR" + echo "ERROR Can't create $MERKLEDIR" + exit 1 + fi +else + logit "INFO" "Workdir found" +fi +# Uncomment next line in case you want to debug the resulting script as well +# echo 'set -xe' > $TEMPORARYDIR/cmd_queue.sh + +# We get the SHA512 hash for the $FILE given +CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}') +FILE_SIZE="$(du -b $FILE | awk '{ print $1 }')" +if [ $FILE_SIZE -lt 4097 ] +then + cp $FILE "$TECHDIR$(basename "$FILE")-00000000000000000000000000000000000000000000000000.chk" +else + # We split the file into 4*1024 bytes and output the chunks into TECHDIR + split -a 50 -b 4096 --additional-suffix ".chk" -d "$FILE" "$TECHDIR$(basename "$FILE")-" +fi + +# We go over there... +cd $TECHDIR +#set -xe +# We get every chunks' SHA512 and we craft a script to rename the chunks and +# move them to CHKDIR +for file in $TEMPORARYDIR/chks/* +do + sha512sum $file >> $TEMPORARYDIR/map +done +# Append last chk if not even number +appendLastIfNotEven(){ + if [ "$(expr $(wc -l "$1" | awk '{ print $1 }') % 2)" != 0 ] + then + tail -n 1 "$1" >> "$1" + fi +} +appendLastIfNotEven "$TEMPORARYDIR/map" +# Figure out how many times we need to pack +totalChunks=`grep 'chk' $TEMPORARYDIR/map | wc -l` +temp="$totalChunks" +timesRan=0 +while [ $temp != 1 ] +do + temp=`expr $temp / 2` + timesRan=`expr $timesRan + 1` +done +printf "Ran %s times \n" "$timesRan" +printf "Total chunks %s \n" "$totalChunks" + +workingIndex="$TEMPORARYDIR/map" +c=$timesRan +while [ "$c" != 0 ] +do + a=1 + printf "Level: %s, will work on %s chunks\n" "$c" "$totalChunks" + while [[ "$a" -lt "$totalChunks" ]] + do + b=`expr "$a" + 1` + sed -n "$a"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b + sed -n "$b"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b + shaSum="$(sha512sum level.$c.pair.$a-$b | awk '{ print $1 }')" + cp level.$c.pair.$a-$b $MERKLEDIR/$shaSum + sha512sum level.$c.pair.$a-$b | awk '{ print $1 }' >> level.$c.map + a=`expr "$a" + 2` + done + workingIndex="level.$c.map" + appendLastIfNotEven "$workingIndex" + shaSum=`sha512sum $workingIndex | awk '{ print $1 }'` + cp $workingIndex $MERKLEDIR/$shaSum + totalChunks=`cat $workingIndex | wc -l` + c=`expr $c - 1` +done + +if [ -f level.1.map ] +then + sha512sum level.1.map +else + echo error + exit 1 +fi + +# Reset file with uniq +cat $TEMPORARYDIR/map | uniq > $TEMPORARYDIR/map2 +cat $TEMPORARYDIR/map2 > $TEMPORARYDIR/map +rm $TEMPORARYDIR/map2 + +counter=0 +while IFS="" read -r p || [ -n "$p" ] +do + printf "mv %s %s%s\n" "$(echo $p | awk '{ print $2 }')" "$CHKDIR" "$(echo $p | awk '{ print $1 }')" >> $TEMPORARYDIR/cmd_queue.sh + counter=`expr "$counter" + 1` +done < $TEMPORARYDIR/map + +# We run the crafted script +sh $TEMPORARYDIR/cmd_queue.sh + +# and we delete it +rm $TEMPORARYDIR/cmd_queue.sh + +# We inform the map about the original $FILE name and SHA512 +echo "$CHECKSUM $(basename "$FILE")" >> $TEMPORARYDIR/map + +# We get the SHA512 hash of the resulted map file +MAPFILEHASH="$(sha512sum $TEMPORARYDIR/map | awk '{ print $1 }')" + +# and we rename it with it and move it to FILEMAPSDIR +`sha512sum $TEMPORARYDIR/map | awk '{print "mv " $2 " '$FILEMAPSDIR'" $1}'` + +# We remove the TEMPORARYDIR +rm -rf $TEMPORARYDIR + +# and print the MAPFILEHASH +echo "$MAPFILEHASH" diff --git a/bin/ak-fs-get b/bin/ak-fs-get new file mode 100755 index 0000000..5959250 --- /dev/null +++ b/bin/ak-fs-get @@ -0,0 +1,104 @@ +#!/bin/bash +# set -xe +if [ -z $1 ] +then + echo "Please provide a SHA512 hash" + exit 1 +fi +if [ -z $2 ] +then + echo "Please an output filename" + exit 2 +fi + +echo $1 | grep "[0123456789abcdef]\{128\}" +if [ $? -ne 0 ] +then + echo "Look, I asked for a SHA512 hash, please try again" + exit 1 +fi +treeRootHash="$1" +outputFilename="$2" + +fmrk="$AK_WORKDIR/fmrk" +ftr="$AK_WORKDIR/ftr" + +# This would be a kind of bootstrap for remote downloads +# +#remoteMrk="http://z.kaotisk-hund.com:8610/mrk" +#remoteTr="http://z.kaotisk-hund.com:8610/tr" +# +pwd>.pwd + +dirWeWere=$(cat .pwd) + +# Enter temp folder +TEMPASSIN="$(ak-tempassin)" +cd $TEMPASSIN + +findDepth(){ + currentNode="$1" + if [ -f $fmrk/$currentNode ] && [ "$(du -b $fmrk/$currentNode | awk '{print $1}')" == "258" ] + then + fileHead="$(head -n 1 $fmrk/$currentNode)" + counter="$(expr $counter + 1)" + findDepth "$fileHead" + elif [ ! -f $fmrk/$currentNode ] + then + printf "%s" "$counter" > depth + else + exit 111 +# Try to download stuff +# wget -s $remoteMrk/$currentNode -O $fmrk/$currentNode +# if [ "$?" -ne 0 ] +# then +# exit 111 +# fi +# findDepth "$currentNode" + fi +} + +currentNode="$treeRootHash" +counter=0 + +printf "%s" "$currentNode" > workspace.0 +findDepth "$currentNode" +depth="$(expr `cat depth` + 1)" +counter="0" +printf "%s" "$depth" +if [ -f output ] +then + rm output +fi +touch output + +while [ "$counter" != "$depth" ] +do + printf "Entering loop... %s %s\n" "$counter" "$depth" + while IFS="" read -r p || [ -n "$p" ] + do + nextLevel="$(expr $counter + 1)" + if [ "$p" == "" ] + then + echo hi + else + if [ -f $fmrk/$p ] + then + if [ "$(head -n 1 $fmrk/$p)" == "$(tail -n 1 $fmrk/$p)" ] + then + head -n 1 $fmrk/$p >> workspace.$nextLevel + else + cat $fmrk/$p >> workspace.$nextLevel + fi + elif [ -f $ftr/$p ] + then + cat $ftr/$p >> output + fi + fi + done < workspace.$counter + counter="$(expr $counter + 1)" +done + +base64 -d output > $dirWeWere/$outputFilename +rm $dirWeWere/.pwd +printf "Recreation of $treeRootHash as $outputFilename succeeded!" diff --git a/bin/ak-sm-merkle-tree b/bin/ak-sm-merkle-tree deleted file mode 100755 index 2ba5e91..0000000 --- a/bin/ak-sm-merkle-tree +++ /dev/null @@ -1,226 +0,0 @@ -#!/bin/bash -# -# The concept is bit more complicated now -# -# 1. For a given file we split in 4KB files inside a temporary directory -# -# 2. We then create a map file and a merkle tree containing the resulted files -# and their sha512sum. -# -# 3. We move the splitted files to our $CHKDIR named after their checksums -# -# 4. We move the merkle tree pairs to $MERKLEDIR -# -# We ultimately want to be seeding the file so -# -# 4. We append the checksum of the original file with its name into the map file -# -# 5. We rename the map file after its checksum and move it to maps directory -# -# 6. We are done! -# - -# Uncomment next line if you want to debug -# set -xe -PROGRAM="$(basename $0)" - -if [ ! -f "$1" ] -then - echo "ERROR File not found" - exit 1 -else - base64 $1 > file - FILE="file" -fi - -# The directory where the chunked data will be living at -CHKDIR="$AK_WORKDIR/ftr/" -# The directory for the map files so we can reconstruct the file -FILEMAPSDIR="$AK_WORKDIR/fmp/" -# Merkle tree file/references -MERKLEDIR="$AK_WORKDIR/fmrk/" -# A temporary root dir to work on -TEMPORARYDIR="$(ak-tempassin)" -# A subdir to split the files there -TECHDIR="$TEMPORARYDIR/chks/" -# A pin point to return from where we came from -CURRENTDIR="$(pwd)/" - -# Our snippet for logging debug info -logit(){ - ak-logthis "$PROGRAM" "$1" "$2" -} - -# Checking directories and create them if necessary -rm -rf $TEMPORARYDIR - -# TECHDIR -if [ ! -d "$TECHDIR" ] -then - mkdir -p "$TECHDIR" - if [ "$?" == 0 ] - then - logit "INFO" "Folder $TECHDIR created!" - else - logit "ERROR" "Problem occured while creating $TECHDIR" - echo "ERROR Can't create $TECHDIR" - exit 1 - fi -else - logit "INFO" "Temp dir found" -fi - -# FILEMAPSDIR -if [ ! -d "$FILEMAPSDIR" ] -then - mkdir -p "$FILEMAPSDIR" - if [ "$?" == 0 ] - then - logit "INFO" "Folder $FILEMAPSDIR created!" - else - logit "ERROR" "Problem occured while creating $FILEMAPSDIR" - echo "ERROR Can't create $FILEMAPSDIR" - exit 1 - fi -else - logit "INFO" "Mapsdir found" -fi - -# CHKDIR -if [ ! -d "$CHKDIR" ] -then - mkdir -p "$CHKDIR" - if [ "$?" == 0 ] - then - logit "INFO" "Folder $CHKDIR created!" - else - logit "ERROR" "Problem occured while creating $CHKDIR" - echo "ERROR Can't create $CHKDIR" - exit 1 - fi -else - logit "INFO" "Workdir found" -fi - -# MERKLEDIR -if [ ! -d "$MERKLEDIR" ] -then - mkdir -p "$MERKLEDIR" - if [ "$?" == 0 ] - then - logit "INFO" "Folder $MERKLEDIR created!" - else - logit "ERROR" "Problem occured while creating $MERKLEDIR" - echo "ERROR Can't create $MERKLEDIR" - exit 1 - fi -else - logit "INFO" "Workdir found" -fi -# Uncomment next line in case you want to debug the resulting script as well -# echo 'set -xe' > $TEMPORARYDIR/cmd_queue.sh - -# We get the SHA512 hash for the $FILE given -CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}') -FILE_SIZE="$(du -b $FILE | awk '{ print $1 }')" -if [ $FILE_SIZE -lt 4097 ] -then - cp $FILE "$TECHDIR$(basename "$FILE")-00000000000000000000000000000000000000000000000000.chk" -else - # We split the file into 4*1024 bytes and output the chunks into TECHDIR - split -a 50 -b 4096 --additional-suffix ".chk" -d "$FILE" "$TECHDIR$(basename "$FILE")-" -fi - -# We go over there... -cd $TECHDIR -#set -xe -# We get every chunks' SHA512 and we craft a script to rename the chunks and -# move them to CHKDIR -for file in $TEMPORARYDIR/chks/* -do - sha512sum $file >> $TEMPORARYDIR/map -done -# Append last chk if not even number -appendLastIfNotEven(){ - if [ "$(expr $(wc -l "$1" | awk '{ print $1 }') % 2)" != 0 ] - then - tail -n 1 "$1" >> "$1" - fi -} -appendLastIfNotEven "$TEMPORARYDIR/map" -# Figure out how many times we need to pack -totalChunks=`grep 'chk' $TEMPORARYDIR/map | wc -l` -temp="$totalChunks" -timesRan=0 -while [ $temp != 1 ] -do - temp=`expr $temp / 2` - timesRan=`expr $timesRan + 1` -done -printf "Ran %s times \n" "$timesRan" -printf "Total chunks %s \n" "$totalChunks" - -workingIndex="$TEMPORARYDIR/map" -c=$timesRan -while [ "$c" != 0 ] -do - a=1 - printf "Level: %s, will work on %s chunks\n" "$c" "$totalChunks" - while [[ "$a" -lt "$totalChunks" ]] - do - b=`expr "$a" + 1` - sed -n "$a"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b - sed -n "$b"p "$workingIndex" | awk '{ print $1 }' >> level.$c.pair.$a-$b - shaSum="$(sha512sum level.$c.pair.$a-$b | awk '{ print $1 }')" - cp level.$c.pair.$a-$b $MERKLEDIR/$shaSum - sha512sum level.$c.pair.$a-$b | awk '{ print $1 }' >> level.$c.map - a=`expr "$a" + 2` - done - workingIndex="level.$c.map" - appendLastIfNotEven "$workingIndex" - shaSum=`sha512sum $workingIndex | awk '{ print $1 }'` - cp $workingIndex $MERKLEDIR/$shaSum - totalChunks=`cat $workingIndex | wc -l` - c=`expr $c - 1` -done - -if [ -f level.1.map ] -then - sha512sum level.1.map -else - echo error - exit 1 -fi - -# Reset file with uniq -cat $TEMPORARYDIR/map | uniq > $TEMPORARYDIR/map2 -cat $TEMPORARYDIR/map2 > $TEMPORARYDIR/map -rm $TEMPORARYDIR/map2 - -counter=0 -while IFS="" read -r p || [ -n "$p" ] -do - printf "mv %s %s%s\n" "$(echo $p | awk '{ print $2 }')" "$CHKDIR" "$(echo $p | awk '{ print $1 }')" >> $TEMPORARYDIR/cmd_queue.sh - counter=`expr "$counter" + 1` -done < $TEMPORARYDIR/map - -# We run the crafted script -sh $TEMPORARYDIR/cmd_queue.sh - -# and we delete it -rm $TEMPORARYDIR/cmd_queue.sh - -# We inform the map about the original $FILE name and SHA512 -echo "$CHECKSUM $(basename "$FILE")" >> $TEMPORARYDIR/map - -# We get the SHA512 hash of the resulted map file -MAPFILEHASH="$(sha512sum $TEMPORARYDIR/map | awk '{ print $1 }')" - -# and we rename it with it and move it to FILEMAPSDIR -`sha512sum $TEMPORARYDIR/map | awk '{print "mv " $2 " '$FILEMAPSDIR'" $1}'` - -# We remove the TEMPORARYDIR -rm -rf $TEMPORARYDIR - -# and print the MAPFILEHASH -echo "$MAPFILEHASH" diff --git a/bin/ak-sm-merkle-tree-to-file b/bin/ak-sm-merkle-tree-to-file deleted file mode 100755 index 5959250..0000000 --- a/bin/ak-sm-merkle-tree-to-file +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -# set -xe -if [ -z $1 ] -then - echo "Please provide a SHA512 hash" - exit 1 -fi -if [ -z $2 ] -then - echo "Please an output filename" - exit 2 -fi - -echo $1 | grep "[0123456789abcdef]\{128\}" -if [ $? -ne 0 ] -then - echo "Look, I asked for a SHA512 hash, please try again" - exit 1 -fi -treeRootHash="$1" -outputFilename="$2" - -fmrk="$AK_WORKDIR/fmrk" -ftr="$AK_WORKDIR/ftr" - -# This would be a kind of bootstrap for remote downloads -# -#remoteMrk="http://z.kaotisk-hund.com:8610/mrk" -#remoteTr="http://z.kaotisk-hund.com:8610/tr" -# -pwd>.pwd - -dirWeWere=$(cat .pwd) - -# Enter temp folder -TEMPASSIN="$(ak-tempassin)" -cd $TEMPASSIN - -findDepth(){ - currentNode="$1" - if [ -f $fmrk/$currentNode ] && [ "$(du -b $fmrk/$currentNode | awk '{print $1}')" == "258" ] - then - fileHead="$(head -n 1 $fmrk/$currentNode)" - counter="$(expr $counter + 1)" - findDepth "$fileHead" - elif [ ! -f $fmrk/$currentNode ] - then - printf "%s" "$counter" > depth - else - exit 111 -# Try to download stuff -# wget -s $remoteMrk/$currentNode -O $fmrk/$currentNode -# if [ "$?" -ne 0 ] -# then -# exit 111 -# fi -# findDepth "$currentNode" - fi -} - -currentNode="$treeRootHash" -counter=0 - -printf "%s" "$currentNode" > workspace.0 -findDepth "$currentNode" -depth="$(expr `cat depth` + 1)" -counter="0" -printf "%s" "$depth" -if [ -f output ] -then - rm output -fi -touch output - -while [ "$counter" != "$depth" ] -do - printf "Entering loop... %s %s\n" "$counter" "$depth" - while IFS="" read -r p || [ -n "$p" ] - do - nextLevel="$(expr $counter + 1)" - if [ "$p" == "" ] - then - echo hi - else - if [ -f $fmrk/$p ] - then - if [ "$(head -n 1 $fmrk/$p)" == "$(tail -n 1 $fmrk/$p)" ] - then - head -n 1 $fmrk/$p >> workspace.$nextLevel - else - cat $fmrk/$p >> workspace.$nextLevel - fi - elif [ -f $ftr/$p ] - then - cat $ftr/$p >> output - fi - fi - done < workspace.$counter - counter="$(expr $counter + 1)" -done - -base64 -d output > $dirWeWere/$outputFilename -rm $dirWeWere/.pwd -printf "Recreation of $treeRootHash as $outputFilename succeeded!" -- cgit v1.2.3