diff options
| -rw-r--r-- | README | 8 | ||||
| -rwxr-xr-x | bin/ak-sm-filejoiner | 88 | ||||
| -rwxr-xr-x | bin/ak-sm-filesplitter | 134 | ||||
| -rwxr-xr-x | lib/_ak_smfiles | 206 | ||||
| -rwxr-xr-x | modules/smfiles/lib.sh (renamed from bin/ak-sm-files) | 82 | ||||
| -rwxr-xr-x | modules/smfiles/main.sh | 56 |
6 files changed, 288 insertions, 286 deletions
@@ -291,7 +291,7 @@ Modules - repositories # Adds repositories to the ZCHAIN - roadmap PROTO Adds a roadmap to the ZCHAIN - transactions # Prototype of transactions in the ZCHAIN - - sm-files # Adds a split file's map to the ZCHAIN + - smfiles # Adds a split file's map to the ZCHAIN - todos PROTO Adds todo list to the ZCHAIN Read more at the `./MODULES` file @@ -302,11 +302,6 @@ Public following (uses ak-profile module) - ak-following # List your follows - ak-unfollow # Unfollow something -File splitters --------------- - - ak-sm-filejoiner # Joins a file from a map - - ak-sm-filesplitter # Splits a file to chunks and creates a map - FS tools -------- - ak-fs @@ -336,6 +331,7 @@ Libraries - lib/_ak_network - lib/_ak_script - lib/_ak_log + - lib/_ak_smfiles IPFS Wrappers ------------- diff --git a/bin/ak-sm-filejoiner b/bin/ak-sm-filejoiner deleted file mode 100755 index f4165f9..0000000 --- a/bin/ak-sm-filejoiner +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -# -set -xe - -PROGRAM="$(basename $0)" - -usage(){ - echo "$PROGRAM <hash>" -} - -cdaw(){ - pwd > tmp_holder -} - -cdaw - -CURDIR="$(cat tmp_holder)" - -TMPWD="/tmp/rjs" - -if [ ! -d "$TMPWD" ] -then - mkdir -p "$TMPWD" - if [ $? -ne 0 ] - then - echo "Can't create $TMPWD dir" - exit 1 - fi -fi - -MAPSDIR="$AK_WORKDIR/fmp" - -CHKDIR="$AK_WORKDIR/ftr" - -cd $CHKDIR -if [ $? -ne 0 ] -then - echo "Can't get dir" - exit 1 -fi - -if [ ! -z $1 ] -then - MAPSFILE="$1" - - echo '#!/bin/bash' > script - - # We create a script to copy all the chunks and rename them to their serialized - # name produced by split when we splitted the file - awk '{print "cp '$CHKDIR'/"$1" '$TMPWD'/"$2" "}' $MAPSDIR/$MAPSFILE| grep chk >> script - - bash script - - if [ $? -ne 0 ] - then - echo "Error executing copy script" - exit 1 - fi - rm script - - cd $TMPWD - - echo "$PWD" - - # Final name we are going to rename to - OUTPUT="$(tail -n1 $MAPSDIR/$MAPSFILE | awk '{print $2}')" - - echo $OUTPUT - - # We grep the MAPSFILE for chk filenames in lines and we print them all in - # one line so the first `cat` will concatenate all chunks to OUTPUT - cat $(echo -n $(cat $MAPSDIR/$MAPSFILE|grep chk|awk '{print $2" "}'|tr -d '\n')) > $OUTPUT - - # We check if everything is okay - sha512sum -c $MAPSDIR/$MAPSFILE - if [ $? -ne 0 ]; then - _ak_log_error "Error while checking sums" - exit 1 - fi - - mv $OUTPUT $CURDIR - - rm -rf "$TMPWD" - - rm $CURDIR/tmp_holder -else - usage -fi diff --git a/bin/ak-sm-filesplitter b/bin/ak-sm-filesplitter deleted file mode 100755 index 4cbb4e4..0000000 --- a/bin/ak-sm-filesplitter +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -# -# The concept is simple -# -# 1. For a given file we split in 1MB files inside a temporary directory -# -# 2. We then create a map file, containing the resulted files and their sha512sum -# -# 3. We move the files to our $CHKDIR named after their checksums -# -# We ultimately want to be seeding the file so -# -# 4. We append the checksum of the original file with its name into the map file -# -# 5. We rename the map file after its checksum and move it to maps directory -# -# 6. We are done! -# - -# Uncomment next line if you want to debug -# set -xe -PROGRAM="$(basename $0)" - -if [ ! -f "$1" ] -then - echo "ERROR File not found" - exit 1 -else - FILE="$1" -fi - -# The directory where the chunked data will be living at -CHKDIR="$AK_WORKDIR/ftr/" -# The directory for the map files so we can reconstruct the file -FILEMAPSDIR="$AK_WORKDIR/fmp/" -# A temporary root dir to work on -TEMPORARYDIR="/tmp/tltmp" -# A subdir to split the files there -TECHDIR="/tmp/tltmp/chks/" -# A pin point to return from where we came from -CURRENTDIR="$(pwd)/" - -# Our snippet for logging debug info -source $AK_LIBDIR/_ak_log - -# Checking directories and create them if necessary - -# TECHDIR -if [ ! -d "$TECHDIR" ] -then - mkdir -p "$TECHDIR" - if [ $? -eq 0 ] - then - _ak_log_info "Folder $TECHDIR created!" - else - _ak_log_error "Problem occured while creating $TECHDIR" - echo "ERROR Can't create $TECHDIR" - exit 1 - fi -else - _ak_log_info "Temp dir found" -fi - -# FILEMAPSDIR -if [ ! -d "$FILEMAPSDIR" ] -then - mkdir -p "$FILEMAPSDIR" - if [ $? -eq 0 ] - then - _ak_log_info "Folder $FILEMAPSDIR created!" - else - _ak_log_error "Problem occured while creating $FILEMAPSDIR" - echo "ERROR Can't create $FILEMAPSDIR" - exit 1 - fi -else - _ak_log_info "Mapsdir found" -fi - -# CHKDIR -if [ ! -d "$CHKDIR" ] -then - mkdir -p "$CHKDIR" - if [ $? -eq 0 ] - then - _ak_log_info "Folder $CHKDIR created!" - else - _ak_log_error "Problem occured while creating $CHKDIR" - echo "ERROR Can't create $CHKDIR" - exit 1 - fi -else - _ak_log_info "Workdir found" -fi - -# Uncomment next line in case you want to debug the resulting script as well -# echo 'set -xe' > $TEMPORARYDIR/cmd_queue.sh - -# We get the SHA512 hash for the $FILE given -CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}') - -# We split the file into 1048576 bytes and output the chunks into TECHDIR -split -b 1048576 --additional-suffix ".chk" -d "$FILE" "$TECHDIR$(basename "$FILE")-" - -# We go over there... -cd $TECHDIR - -# We get every chunks' SHA512 and we craft a script to rename the chunks and -# move them to CHKDIR -sha512sum * > $TEMPORARYDIR/map; while IFS="" read -r p || [ -n "$p" ] -do - echo $p | awk '{print "mv " $2 " '$CHKDIR'" $1}' >> $TEMPORARYDIR/cmd_queue.sh -done < $TEMPORARYDIR/map - -# We run the crafted script -sh $TEMPORARYDIR/cmd_queue.sh - -# and we delete it -rm $TEMPORARYDIR/cmd_queue.sh - -# We inform the map about the original $FILE name and SHA512 -echo "$CHECKSUM $(basename "$FILE")" >> $TEMPORARYDIR/map - -# We get the SHA512 hash of the resulted map file -MAPFILEHASH="$(sha512sum $TEMPORARYDIR/map | awk '{ print $1 }')" - -# and we rename it with it and move it to FILEMAPSDIR -`sha512sum $TEMPORARYDIR/map | awk '{print "mv " $2 " '$FILEMAPSDIR'" $1}'` - -# We remove the TEMPORARYDIR -rm -rf $TEMPORARYDIR - -# and print the MAPFILEHASH -echo "$MAPFILEHASH" diff --git a/lib/_ak_smfiles b/lib/_ak_smfiles new file mode 100755 index 0000000..55ec381 --- /dev/null +++ b/lib/_ak_smfiles @@ -0,0 +1,206 @@ +#!/bin/bash +source $AK_LIBDIR/_ak_log + +# The directory where the chunked data will be living at +CHKDIR="$AK_WORKDIR/ftr" +# The directory for the map files so we can reconstruct the file +FILEMAPSDIR="$AK_WORKDIR/fmp" +# A temporary root dir to work on +TEMPORARYDIR="/tmp/tltmp" +# A subdir to split the files there +TECHDIR="/tmp/tltmp/chks" +# A pin point to return from where we came from +CURRENTDIR="$(pwd)/" +# Checking directories and create them if necessary + +if [ ! -d "$TECHDIR" ] +then + mkdir -p "$TECHDIR" + if [ $? -eq 0 ] + then + _ak_log_info "Folder $TECHDIR created!" + else + _ak_log_error "Problem occured while creating $TECHDIR" + exit 1 + fi +else + _ak_log_info "Temp dir found" +fi + +if [ ! -d "$FILEMAPSDIR" ] +then + mkdir -p "$FILEMAPSDIR" + if [ $? -eq 0 ] + then + _ak_log_info "Folder $FILEMAPSDIR created!" + else + _ak_log_error "Problem occured while creating $FILEMAPSDIR" + exit 1 + fi +else + _ak_log_info "Mapsdir found" +fi + +if [ ! -d "$CHKDIR" ] +then + mkdir -p "$CHKDIR" + if [ $? -eq 0 ] + then + _ak_log_info "Folder $CHKDIR created!" + else + _ak_log_error "Problem occured while creating $CHKDIR" + exit 1 + fi +else + _ak_log_info "Workdir found" +fi + +cdaw(){ + pwd > tmp_holder +} + +_ak_sm_file_joiner(){ + cdaw + + CURDIR="$(cat tmp_holder)" + + TMPWD="/tmp/rjs" + + if [ ! -d "$TMPWD" ] + then + mkdir -p "$TMPWD" + if [ $? -ne 0 ] + then + _ak_log_error "Can't create $TMPWD dir" + exit 1 + fi + fi + + cd $CHKDIR + if [ $? -ne 0 ] + then + _ak_log_error "Can't get dir" + exit 1 + fi + + if [ ! -z $1 ] + then + MAPSFILE="$1" + + echo '#!/bin/bash' > script + + # We create a script to copy all the chunks and rename them to their serialized + # name produced by split when we splitted the file + awk '{print "cp '$CHKDIR'/"$1" '$TMPWD'/"$2" "}' $FILEMAPSDIR/$MAPSFILE| grep chk >> script + + bash script + + if [ $? -ne 0 ] + then + echo "Error executing copy script" + exit 1 + fi + rm script + + cd $TMPWD + + echo "$PWD" + + # Final name we are going to rename to + OUTPUT="$(tail -n 1 $FILEMAPSDIR/$MAPSFILE | awk '{print $2}')" + + echo $OUTPUT + + # We grep the MAPSFILE for chk filenames in lines and we print them all in + # one line so the first `cat` will concatenate all chunks to OUTPUT + cat $(echo -n $(cat $FILEMAPSDIR/$MAPSFILE|grep chk|awk '{print $2" "}'|tr -d '\n')) > $OUTPUT + + # We check if everything is okay + sha512sum -c $FILEMAPSDIR/$MAPSFILE + if [ $? -ne 0 ] + then + _ak_log_error "Error while checking sums" + exit 1 + fi + + mv $OUTPUT $CURDIR + + rm -rf "$TMPWD" + + rm $CURDIR/tmp_holder + else + _ak_log_error "No hash given" + exit 1 + fi +} + +_ak_sm_file_splitter(){ + # + # The concept is simple + # + # 1. For a given file we split in 1MB files inside a temporary directory + # + # 2. We then create a map file, containing the resulted files and their sha512sum + # + # 3. We move the files to our $CHKDIR named after their checksums + # + # We ultimately want to be seeding the file so + # + # 4. We append the checksum of the original file with its name into the map file + # + # 5. We rename the map file after its checksum and move it to maps directory + # + # 6. We are done! + # + + # Uncomment next line if you want to debug + # set -xe + + if [ ! -f "$1" ] + then + echo "ERROR File not found" + exit 1 + else + FILE="$1" + fi + + # Uncomment next line in case you want to debug the resulting script as well + # echo 'set -xe' > $TEMPORARYDIR/cmd_queue.sh + + # We get the SHA512 hash for the $FILE given + CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}') + + # We split the file into 1048576 bytes and output the chunks into TECHDIR + split -b 1048576 --additional-suffix ".chk" -d "$FILE" "$TECHDIR/$(basename "$FILE")-" + + # We go over there... + cd $TECHDIR + + # We get every chunks' SHA512 and we craft a script to rename the chunks and + # move them to CHKDIR + sha512sum * > $TEMPORARYDIR/map; while IFS="" read -r p || [ -n "$p" ] + do + echo $p | awk '{print "mv " $2 " '$CHKDIR'/" $1}' >> $TEMPORARYDIR/cmd_queue.sh + done < $TEMPORARYDIR/map + + # We run the crafted script + sh $TEMPORARYDIR/cmd_queue.sh + + # and we delete it + rm $TEMPORARYDIR/cmd_queue.sh + + # We inform the map about the original $FILE name and SHA512 + echo "$CHECKSUM $(basename "$FILE")" >> $TEMPORARYDIR/map + + # We get the SHA512 hash of the resulted map file + MAPFILEHASH="$(sha512sum $TEMPORARYDIR/map | awk '{ print $1 }')" + + # and we rename it with it and move it to FILEMAPSDIR + `sha512sum $TEMPORARYDIR/map | awk '{print "mv " $2 " '$FILEMAPSDIR'/" $1}'` + + # We remove the TEMPORARYDIR + rm -rf $TEMPORARYDIR + + # and print the MAPFILEHASH + echo "$MAPFILEHASH" +} diff --git a/bin/ak-sm-files b/modules/smfiles/lib.sh index 317149e..ef88278 100755 --- a/bin/ak-sm-files +++ b/modules/smfiles/lib.sh @@ -1,41 +1,10 @@ #!/bin/bash -## sm files -## -## -h, --help Prints this help message -## -## --add <file> Adds file to zchain as a zblock -## -## --index List files -## -## --full-index List all files -## -## --ls-map-files List map files -## -ZFILESDIR="$AK_WORKDIR/files" -pwd > .pwd -CRD=$(cat .pwd) - -PROGRAM="$(basename $0)" -#set -xe + source $AK_LIBDIR/_ak_log source $AK_LIBDIR/_ak_ipfs source $AK_LIBDIR/_ak_gpg source $AK_LIBDIR/_ak_zblock - -if [ ! -d $ZFILESDIR ]; then - mkdir $ZFILESDIR - if [ $? == 0 ] - then - _ak_log_info "Folder $ZFILESDIR created!" - else - _ak_log_error "Failed to create $ZFILESDIR folder" - exit 1 - fi - cd $ZFILESDIR -else - _ak_log_info "$ZFILESDIR found!" -fi - +source $AK_LIBDIR/_ak_smfiles _ak_sm_files_add(){ FILENAME="$1" @@ -51,7 +20,8 @@ _ak_sm_files_main(){ echo "Adding $FILENAME" _ak_log_info "Switching to tmp folder..." - if [ $? == 0 ]; then + if [ $? -eq 0 ] + then _ak_log_info "Success" else _ak_log_error "Error with tmp folder" @@ -60,7 +30,8 @@ _ak_sm_files_main(){ _ak_log_info "Copying $1 to $TEMPASSIN" cp $CRP/$FILENAME $FILENAME - if [ $? == 0 ]; then + if [ $? -eq 0 ] + then _ak_log_info "Copied successfully" else _ak_log_error "Error copying..." @@ -68,15 +39,17 @@ _ak_sm_files_main(){ _ak_log_info "Adding $FILENAME to IPFS..." FILE_IPFS_HASH=$(_ak_ipfs_add $FILENAME) - if [ $? == 0 ]; then + if [ $? -eq 0 ] + then _ak_log_info "Added $FILENAME to IPFS" else _ak_log_error "Error in adding the $FILENAME to IPFS" fi _ak_log_info "Adding $FILE to SHAMAPSYS..." - FILEMAP_SHA512_HASH=$(ak-sm-filesplitter $FILENAME) - if [ $? == 0 ]; then + FILEMAP_SHA512_HASH=$(_ak_sm_file_splitter $FILENAME) + if [ $? -eq 0 ] + then _ak_log_info "Added $FILENAME to SHAMAPSYS" else _ak_log_error "Error in adding the $FILENAME to SHAMAPSYS" @@ -85,7 +58,8 @@ _ak_sm_files_main(){ _ak_log_info "Signing..." SIGN_FILE=$FILENAME".asc" _ak_gpg_sign_detached $SIGN_FILE $FILENAME - if [ $? == 0 ]; then + if [ $? -eq 0 ] + then _ak_log_info "Signed" else _ak_log_error "Error while signing" @@ -93,15 +67,17 @@ _ak_sm_files_main(){ _ak_log_info "Adding signature to IPFS" SIGNATURE=$(_ak_ipfs_add $SIGN_FILE) - if [ $? == 0 ]; then + if [ $? -eq 0 ] + then _ak_log_info "Added" else _ak_log_error "Error while adding" fi _ak_log_info "Adding signature to SHAMAPSYS" - SHAMAPSIGMAP=$(ak-sm-filesplitter $SIGN_FILE) - if [ $? == 0 ]; then + SHAMAPSIGMAP=$(_ak_sm_file_splitter $SIGN_FILE) + if [ $? -eq 0 ] + then _ak_log_info "Added" else _ak_log_error "Error while adding" @@ -123,7 +99,7 @@ EOF echo "Publishing..." _ak_zblock_pack sha-files/announce $(pwd)/data - if [ $? == 0 ] + if [ $? -eq 0 ] then echo "cool" else @@ -138,26 +114,16 @@ _ak_sm_files_index(){ _ak_sm_files_ls_mapfiles(){ cd $AK_WORKDIR/fmp - for f in `find . -type f | sed -e 's/\.\///g'`;do - FILENAME="$(tail -n1 $f | grep '^[abcdef1234567890]' | awk '{ print $2 }')" - FILEHASH="$(tail -n1 $f | grep '^[abcdef1234567890]' | awk '{ print $1 }')" + for f in `find . -type f | sed -e 's/\.\///g'` + do + FILENAME="$(tail -n 1 $f | grep '^[abcdef1234567890]' | awk '{ print $2 }')" + FILEHASH="$(tail -n 1 $f | grep '^[abcdef1234567890]' | awk '{ print $1 }')" MAPFILE="$f" printf "\nMap: %s\nFilename: %s\nSum: %s\n\n" $MAPFILE $FILENAME $FILEHASH done } _ak_sm_files_full_index(){ - tail -n1 $AK_WORKDIR/fmp/* | grep '^[abcdef1234567890]' + tail -n 1 $AK_WORKDIR/fmp/* | grep '^[abcdef1234567890]' } -if [ ! -z $1 ]; then - case $1 in - -h | --help) usage; exit;; - --add) _ak_sm_files_add $2; exit;; - --index) _ak_sm_files_index; exit;; - --full-index) _ak_sm_files_full_index; exit;; - --ls-map-files) _ak_sm_files_ls_mapfiles; exit;; - *) usage; exit;; - esac -else usage -fi diff --git a/modules/smfiles/main.sh b/modules/smfiles/main.sh new file mode 100755 index 0000000..f7662c3 --- /dev/null +++ b/modules/smfiles/main.sh @@ -0,0 +1,56 @@ +#!/bin/bash +## sm files +## +## -h, --help Prints this help message +## +## --add <file> Adds file to zchain as a zblock +## +## --index List files +## +## --full-index List all files +## +## --ls-map-files List map files +## +ZFILESDIR="$AK_WORKDIR/files" +pwd > .pwd +CRD=$(cat .pwd) + +fullprogrampath="$(realpath $0)" +PROGRAM="$(realpath $0 | rev |cut -d '/' -f 2 | rev)" +descriptionString="Module to files in zchain" + +source $AK_LIBDIR/_ak_log +source $AK_LIBDIR/_ak_script +source $AK_MODULESDIR/$PROGRAM/lib.sh + +if [ ! -d $ZFILESDIR ] +then + mkdir $ZFILESDIR + if [ $? -eq 0 ] + then + _ak_log_info "Folder $ZFILESDIR created!" + else + _ak_log_error "Failed to create $ZFILESDIR folder" + exit 1 + fi + cd $ZFILESDIR +else + _ak_log_info "$ZFILESDIR found!" +fi + + + +if [ ! -z $1 ] +then + case $1 in + -h | --help) _ak_usage; exit;; + --add) _ak_sm_files_add $2; exit;; + --index) _ak_sm_files_index; exit;; + --full-index) _ak_sm_files_full_index; exit;; + --ls-map-files) _ak_sm_files_ls_mapfiles; exit;; + *) _ak_usage; exit;; + esac +else + _ak_usage +fi + |