From 38ad449f562af439640563112e2dc38fe2a960cd Mon Sep 17 00:00:00 2001 From: kaotisk Date: Tue, 16 Jul 2024 05:24:52 +0300 Subject: Moved `ak-sm-files` to module `smfiles` Also, cleaning up the `bin/` directory: - ak-sm-filejoiner and - ak-sm-filesplitter are now part of `lib/_ak_smfiles` Updated README as well removing the above scripts and fixing the typo: - sm-files -> smfiles from the modules list. Note that the module part, as well as the `files` module are going to be removed in favor of `ak-fs`. Stay tuned for that. To update, do: ``` git pull ./update.sh ``` --- README | 8 +- bin/ak-sm-filejoiner | 88 --------------------- bin/ak-sm-files | 163 -------------------------------------- bin/ak-sm-filesplitter | 134 ------------------------------- lib/_ak_smfiles | 206 ++++++++++++++++++++++++++++++++++++++++++++++++ modules/smfiles/lib.sh | 129 ++++++++++++++++++++++++++++++ modules/smfiles/main.sh | 56 +++++++++++++ 7 files changed, 393 insertions(+), 391 deletions(-) delete mode 100755 bin/ak-sm-filejoiner delete mode 100755 bin/ak-sm-files delete mode 100755 bin/ak-sm-filesplitter create mode 100755 lib/_ak_smfiles create mode 100755 modules/smfiles/lib.sh create mode 100755 modules/smfiles/main.sh diff --git a/README b/README index 69f0cd2..b105b7a 100644 --- a/README +++ b/README @@ -291,7 +291,7 @@ Modules - repositories # Adds repositories to the ZCHAIN - roadmap PROTO Adds a roadmap to the ZCHAIN - transactions # Prototype of transactions in the ZCHAIN - - sm-files # Adds a split file's map to the ZCHAIN + - smfiles # Adds a split file's map to the ZCHAIN - todos PROTO Adds todo list to the ZCHAIN Read more at the `./MODULES` file @@ -302,11 +302,6 @@ Public following (uses ak-profile module) - ak-following # List your follows - ak-unfollow # Unfollow something -File splitters --------------- - - ak-sm-filejoiner # Joins a file from a map - - ak-sm-filesplitter # Splits a file to chunks and creates a map - FS tools -------- - ak-fs @@ -336,6 +331,7 @@ Libraries - lib/_ak_network - lib/_ak_script - lib/_ak_log + - lib/_ak_smfiles IPFS Wrappers ------------- diff --git a/bin/ak-sm-filejoiner b/bin/ak-sm-filejoiner deleted file mode 100755 index f4165f9..0000000 --- a/bin/ak-sm-filejoiner +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -# -set -xe - -PROGRAM="$(basename $0)" - -usage(){ - echo "$PROGRAM " -} - -cdaw(){ - pwd > tmp_holder -} - -cdaw - -CURDIR="$(cat tmp_holder)" - -TMPWD="/tmp/rjs" - -if [ ! -d "$TMPWD" ] -then - mkdir -p "$TMPWD" - if [ $? -ne 0 ] - then - echo "Can't create $TMPWD dir" - exit 1 - fi -fi - -MAPSDIR="$AK_WORKDIR/fmp" - -CHKDIR="$AK_WORKDIR/ftr" - -cd $CHKDIR -if [ $? -ne 0 ] -then - echo "Can't get dir" - exit 1 -fi - -if [ ! -z $1 ] -then - MAPSFILE="$1" - - echo '#!/bin/bash' > script - - # We create a script to copy all the chunks and rename them to their serialized - # name produced by split when we splitted the file - awk '{print "cp '$CHKDIR'/"$1" '$TMPWD'/"$2" "}' $MAPSDIR/$MAPSFILE| grep chk >> script - - bash script - - if [ $? -ne 0 ] - then - echo "Error executing copy script" - exit 1 - fi - rm script - - cd $TMPWD - - echo "$PWD" - - # Final name we are going to rename to - OUTPUT="$(tail -n1 $MAPSDIR/$MAPSFILE | awk '{print $2}')" - - echo $OUTPUT - - # We grep the MAPSFILE for chk filenames in lines and we print them all in - # one line so the first `cat` will concatenate all chunks to OUTPUT - cat $(echo -n $(cat $MAPSDIR/$MAPSFILE|grep chk|awk '{print $2" "}'|tr -d '\n')) > $OUTPUT - - # We check if everything is okay - sha512sum -c $MAPSDIR/$MAPSFILE - if [ $? -ne 0 ]; then - _ak_log_error "Error while checking sums" - exit 1 - fi - - mv $OUTPUT $CURDIR - - rm -rf "$TMPWD" - - rm $CURDIR/tmp_holder -else - usage -fi diff --git a/bin/ak-sm-files b/bin/ak-sm-files deleted file mode 100755 index 317149e..0000000 --- a/bin/ak-sm-files +++ /dev/null @@ -1,163 +0,0 @@ -#!/bin/bash -## sm files -## -## -h, --help Prints this help message -## -## --add Adds file to zchain as a zblock -## -## --index List files -## -## --full-index List all files -## -## --ls-map-files List map files -## -ZFILESDIR="$AK_WORKDIR/files" -pwd > .pwd -CRD=$(cat .pwd) - -PROGRAM="$(basename $0)" -#set -xe -source $AK_LIBDIR/_ak_log -source $AK_LIBDIR/_ak_ipfs -source $AK_LIBDIR/_ak_gpg -source $AK_LIBDIR/_ak_zblock - -if [ ! -d $ZFILESDIR ]; then - mkdir $ZFILESDIR - if [ $? == 0 ] - then - _ak_log_info "Folder $ZFILESDIR created!" - else - _ak_log_error "Failed to create $ZFILESDIR folder" - exit 1 - fi - cd $ZFILESDIR -else - _ak_log_info "$ZFILESDIR found!" -fi - - -_ak_sm_files_add(){ - FILENAME="$1" - _ak_sm_files_main $FILENAME $CRD - cat data | jq -M -} -_ak_sm_files_main(){ - FILENAME="$1" - CRP="$2" - - TEMPASSIN="$(_ak_make_temp_directory)" - cd $TEMPASSIN - - echo "Adding $FILENAME" - _ak_log_info "Switching to tmp folder..." - if [ $? == 0 ]; then - _ak_log_info "Success" - else - _ak_log_error "Error with tmp folder" - exit 5 - fi - _ak_log_info "Copying $1 to $TEMPASSIN" - - cp $CRP/$FILENAME $FILENAME - if [ $? == 0 ]; then - _ak_log_info "Copied successfully" - else - _ak_log_error "Error copying..." - fi - - _ak_log_info "Adding $FILENAME to IPFS..." - FILE_IPFS_HASH=$(_ak_ipfs_add $FILENAME) - if [ $? == 0 ]; then - _ak_log_info "Added $FILENAME to IPFS" - else - _ak_log_error "Error in adding the $FILENAME to IPFS" - fi - - _ak_log_info "Adding $FILE to SHAMAPSYS..." - FILEMAP_SHA512_HASH=$(ak-sm-filesplitter $FILENAME) - if [ $? == 0 ]; then - _ak_log_info "Added $FILENAME to SHAMAPSYS" - else - _ak_log_error "Error in adding the $FILENAME to SHAMAPSYS" - fi - - _ak_log_info "Signing..." - SIGN_FILE=$FILENAME".asc" - _ak_gpg_sign_detached $SIGN_FILE $FILENAME - if [ $? == 0 ]; then - _ak_log_info "Signed" - else - _ak_log_error "Error while signing" - fi - - _ak_log_info "Adding signature to IPFS" - SIGNATURE=$(_ak_ipfs_add $SIGN_FILE) - if [ $? == 0 ]; then - _ak_log_info "Added" - else - _ak_log_error "Error while adding" - fi - - _ak_log_info "Adding signature to SHAMAPSYS" - SHAMAPSIGMAP=$(ak-sm-filesplitter $SIGN_FILE) - if [ $? == 0 ]; then - _ak_log_info "Added" - else - _ak_log_error "Error while adding" - fi - - cat > data < $TEMPORARYDIR/cmd_queue.sh - -# We get the SHA512 hash for the $FILE given -CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}') - -# We split the file into 1048576 bytes and output the chunks into TECHDIR -split -b 1048576 --additional-suffix ".chk" -d "$FILE" "$TECHDIR$(basename "$FILE")-" - -# We go over there... -cd $TECHDIR - -# We get every chunks' SHA512 and we craft a script to rename the chunks and -# move them to CHKDIR -sha512sum * > $TEMPORARYDIR/map; while IFS="" read -r p || [ -n "$p" ] -do - echo $p | awk '{print "mv " $2 " '$CHKDIR'" $1}' >> $TEMPORARYDIR/cmd_queue.sh -done < $TEMPORARYDIR/map - -# We run the crafted script -sh $TEMPORARYDIR/cmd_queue.sh - -# and we delete it -rm $TEMPORARYDIR/cmd_queue.sh - -# We inform the map about the original $FILE name and SHA512 -echo "$CHECKSUM $(basename "$FILE")" >> $TEMPORARYDIR/map - -# We get the SHA512 hash of the resulted map file -MAPFILEHASH="$(sha512sum $TEMPORARYDIR/map | awk '{ print $1 }')" - -# and we rename it with it and move it to FILEMAPSDIR -`sha512sum $TEMPORARYDIR/map | awk '{print "mv " $2 " '$FILEMAPSDIR'" $1}'` - -# We remove the TEMPORARYDIR -rm -rf $TEMPORARYDIR - -# and print the MAPFILEHASH -echo "$MAPFILEHASH" diff --git a/lib/_ak_smfiles b/lib/_ak_smfiles new file mode 100755 index 0000000..55ec381 --- /dev/null +++ b/lib/_ak_smfiles @@ -0,0 +1,206 @@ +#!/bin/bash +source $AK_LIBDIR/_ak_log + +# The directory where the chunked data will be living at +CHKDIR="$AK_WORKDIR/ftr" +# The directory for the map files so we can reconstruct the file +FILEMAPSDIR="$AK_WORKDIR/fmp" +# A temporary root dir to work on +TEMPORARYDIR="/tmp/tltmp" +# A subdir to split the files there +TECHDIR="/tmp/tltmp/chks" +# A pin point to return from where we came from +CURRENTDIR="$(pwd)/" +# Checking directories and create them if necessary + +if [ ! -d "$TECHDIR" ] +then + mkdir -p "$TECHDIR" + if [ $? -eq 0 ] + then + _ak_log_info "Folder $TECHDIR created!" + else + _ak_log_error "Problem occured while creating $TECHDIR" + exit 1 + fi +else + _ak_log_info "Temp dir found" +fi + +if [ ! -d "$FILEMAPSDIR" ] +then + mkdir -p "$FILEMAPSDIR" + if [ $? -eq 0 ] + then + _ak_log_info "Folder $FILEMAPSDIR created!" + else + _ak_log_error "Problem occured while creating $FILEMAPSDIR" + exit 1 + fi +else + _ak_log_info "Mapsdir found" +fi + +if [ ! -d "$CHKDIR" ] +then + mkdir -p "$CHKDIR" + if [ $? -eq 0 ] + then + _ak_log_info "Folder $CHKDIR created!" + else + _ak_log_error "Problem occured while creating $CHKDIR" + exit 1 + fi +else + _ak_log_info "Workdir found" +fi + +cdaw(){ + pwd > tmp_holder +} + +_ak_sm_file_joiner(){ + cdaw + + CURDIR="$(cat tmp_holder)" + + TMPWD="/tmp/rjs" + + if [ ! -d "$TMPWD" ] + then + mkdir -p "$TMPWD" + if [ $? -ne 0 ] + then + _ak_log_error "Can't create $TMPWD dir" + exit 1 + fi + fi + + cd $CHKDIR + if [ $? -ne 0 ] + then + _ak_log_error "Can't get dir" + exit 1 + fi + + if [ ! -z $1 ] + then + MAPSFILE="$1" + + echo '#!/bin/bash' > script + + # We create a script to copy all the chunks and rename them to their serialized + # name produced by split when we splitted the file + awk '{print "cp '$CHKDIR'/"$1" '$TMPWD'/"$2" "}' $FILEMAPSDIR/$MAPSFILE| grep chk >> script + + bash script + + if [ $? -ne 0 ] + then + echo "Error executing copy script" + exit 1 + fi + rm script + + cd $TMPWD + + echo "$PWD" + + # Final name we are going to rename to + OUTPUT="$(tail -n 1 $FILEMAPSDIR/$MAPSFILE | awk '{print $2}')" + + echo $OUTPUT + + # We grep the MAPSFILE for chk filenames in lines and we print them all in + # one line so the first `cat` will concatenate all chunks to OUTPUT + cat $(echo -n $(cat $FILEMAPSDIR/$MAPSFILE|grep chk|awk '{print $2" "}'|tr -d '\n')) > $OUTPUT + + # We check if everything is okay + sha512sum -c $FILEMAPSDIR/$MAPSFILE + if [ $? -ne 0 ] + then + _ak_log_error "Error while checking sums" + exit 1 + fi + + mv $OUTPUT $CURDIR + + rm -rf "$TMPWD" + + rm $CURDIR/tmp_holder + else + _ak_log_error "No hash given" + exit 1 + fi +} + +_ak_sm_file_splitter(){ + # + # The concept is simple + # + # 1. For a given file we split in 1MB files inside a temporary directory + # + # 2. We then create a map file, containing the resulted files and their sha512sum + # + # 3. We move the files to our $CHKDIR named after their checksums + # + # We ultimately want to be seeding the file so + # + # 4. We append the checksum of the original file with its name into the map file + # + # 5. We rename the map file after its checksum and move it to maps directory + # + # 6. We are done! + # + + # Uncomment next line if you want to debug + # set -xe + + if [ ! -f "$1" ] + then + echo "ERROR File not found" + exit 1 + else + FILE="$1" + fi + + # Uncomment next line in case you want to debug the resulting script as well + # echo 'set -xe' > $TEMPORARYDIR/cmd_queue.sh + + # We get the SHA512 hash for the $FILE given + CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}') + + # We split the file into 1048576 bytes and output the chunks into TECHDIR + split -b 1048576 --additional-suffix ".chk" -d "$FILE" "$TECHDIR/$(basename "$FILE")-" + + # We go over there... + cd $TECHDIR + + # We get every chunks' SHA512 and we craft a script to rename the chunks and + # move them to CHKDIR + sha512sum * > $TEMPORARYDIR/map; while IFS="" read -r p || [ -n "$p" ] + do + echo $p | awk '{print "mv " $2 " '$CHKDIR'/" $1}' >> $TEMPORARYDIR/cmd_queue.sh + done < $TEMPORARYDIR/map + + # We run the crafted script + sh $TEMPORARYDIR/cmd_queue.sh + + # and we delete it + rm $TEMPORARYDIR/cmd_queue.sh + + # We inform the map about the original $FILE name and SHA512 + echo "$CHECKSUM $(basename "$FILE")" >> $TEMPORARYDIR/map + + # We get the SHA512 hash of the resulted map file + MAPFILEHASH="$(sha512sum $TEMPORARYDIR/map | awk '{ print $1 }')" + + # and we rename it with it and move it to FILEMAPSDIR + `sha512sum $TEMPORARYDIR/map | awk '{print "mv " $2 " '$FILEMAPSDIR'/" $1}'` + + # We remove the TEMPORARYDIR + rm -rf $TEMPORARYDIR + + # and print the MAPFILEHASH + echo "$MAPFILEHASH" +} diff --git a/modules/smfiles/lib.sh b/modules/smfiles/lib.sh new file mode 100755 index 0000000..ef88278 --- /dev/null +++ b/modules/smfiles/lib.sh @@ -0,0 +1,129 @@ +#!/bin/bash + +source $AK_LIBDIR/_ak_log +source $AK_LIBDIR/_ak_ipfs +source $AK_LIBDIR/_ak_gpg +source $AK_LIBDIR/_ak_zblock +source $AK_LIBDIR/_ak_smfiles + +_ak_sm_files_add(){ + FILENAME="$1" + _ak_sm_files_main $FILENAME $CRD + cat data | jq -M +} +_ak_sm_files_main(){ + FILENAME="$1" + CRP="$2" + + TEMPASSIN="$(_ak_make_temp_directory)" + cd $TEMPASSIN + + echo "Adding $FILENAME" + _ak_log_info "Switching to tmp folder..." + if [ $? -eq 0 ] + then + _ak_log_info "Success" + else + _ak_log_error "Error with tmp folder" + exit 5 + fi + _ak_log_info "Copying $1 to $TEMPASSIN" + + cp $CRP/$FILENAME $FILENAME + if [ $? -eq 0 ] + then + _ak_log_info "Copied successfully" + else + _ak_log_error "Error copying..." + fi + + _ak_log_info "Adding $FILENAME to IPFS..." + FILE_IPFS_HASH=$(_ak_ipfs_add $FILENAME) + if [ $? -eq 0 ] + then + _ak_log_info "Added $FILENAME to IPFS" + else + _ak_log_error "Error in adding the $FILENAME to IPFS" + fi + + _ak_log_info "Adding $FILE to SHAMAPSYS..." + FILEMAP_SHA512_HASH=$(_ak_sm_file_splitter $FILENAME) + if [ $? -eq 0 ] + then + _ak_log_info "Added $FILENAME to SHAMAPSYS" + else + _ak_log_error "Error in adding the $FILENAME to SHAMAPSYS" + fi + + _ak_log_info "Signing..." + SIGN_FILE=$FILENAME".asc" + _ak_gpg_sign_detached $SIGN_FILE $FILENAME + if [ $? -eq 0 ] + then + _ak_log_info "Signed" + else + _ak_log_error "Error while signing" + fi + + _ak_log_info "Adding signature to IPFS" + SIGNATURE=$(_ak_ipfs_add $SIGN_FILE) + if [ $? -eq 0 ] + then + _ak_log_info "Added" + else + _ak_log_error "Error while adding" + fi + + _ak_log_info "Adding signature to SHAMAPSYS" + SHAMAPSIGMAP=$(_ak_sm_file_splitter $SIGN_FILE) + if [ $? -eq 0 ] + then + _ak_log_info "Added" + else + _ak_log_error "Error while adding" + fi + + cat > data < Adds file to zchain as a zblock +## +## --index List files +## +## --full-index List all files +## +## --ls-map-files List map files +## +ZFILESDIR="$AK_WORKDIR/files" +pwd > .pwd +CRD=$(cat .pwd) + +fullprogrampath="$(realpath $0)" +PROGRAM="$(realpath $0 | rev |cut -d '/' -f 2 | rev)" +descriptionString="Module to files in zchain" + +source $AK_LIBDIR/_ak_log +source $AK_LIBDIR/_ak_script +source $AK_MODULESDIR/$PROGRAM/lib.sh + +if [ ! -d $ZFILESDIR ] +then + mkdir $ZFILESDIR + if [ $? -eq 0 ] + then + _ak_log_info "Folder $ZFILESDIR created!" + else + _ak_log_error "Failed to create $ZFILESDIR folder" + exit 1 + fi + cd $ZFILESDIR +else + _ak_log_info "$ZFILESDIR found!" +fi + + + +if [ ! -z $1 ] +then + case $1 in + -h | --help) _ak_usage; exit;; + --add) _ak_sm_files_add $2; exit;; + --index) _ak_sm_files_index; exit;; + --full-index) _ak_sm_files_full_index; exit;; + --ls-map-files) _ak_sm_files_ls_mapfiles; exit;; + *) _ak_usage; exit;; + esac +else + _ak_usage +fi + -- cgit v1.2.3