aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorkaotisk <kaotisk@arching-kaos.org>2024-07-16 05:24:52 +0300
committerkaotisk <kaotisk@arching-kaos.org>2024-07-16 05:24:52 +0300
commit38ad449f562af439640563112e2dc38fe2a960cd (patch)
treeb3d957e61d0de180e258d0479a9d11e51caa55d8
parent887f912491cdeaa1c1a8b4d13d48c46909f391b7 (diff)
downloadarching-kaos-tools-38ad449f562af439640563112e2dc38fe2a960cd.tar.gz
arching-kaos-tools-38ad449f562af439640563112e2dc38fe2a960cd.tar.bz2
arching-kaos-tools-38ad449f562af439640563112e2dc38fe2a960cd.zip
Moved `ak-sm-files` to module `smfiles`
Also, cleaning up the `bin/` directory: - ak-sm-filejoiner and - ak-sm-filesplitter are now part of `lib/_ak_smfiles` Updated README as well removing the above scripts and fixing the typo: - sm-files -> smfiles from the modules list. Note that the module part, as well as the `files` module are going to be removed in favor of `ak-fs`. Stay tuned for that. To update, do: ``` git pull ./update.sh ```
-rw-r--r--README8
-rwxr-xr-xbin/ak-sm-filejoiner88
-rwxr-xr-xbin/ak-sm-filesplitter134
-rwxr-xr-xlib/_ak_smfiles206
-rwxr-xr-xmodules/smfiles/lib.sh (renamed from bin/ak-sm-files)82
-rwxr-xr-xmodules/smfiles/main.sh56
6 files changed, 288 insertions, 286 deletions
diff --git a/README b/README
index 69f0cd2..b105b7a 100644
--- a/README
+++ b/README
@@ -291,7 +291,7 @@ Modules
- repositories # Adds repositories to the ZCHAIN
- roadmap PROTO Adds a roadmap to the ZCHAIN
- transactions # Prototype of transactions in the ZCHAIN
- - sm-files # Adds a split file's map to the ZCHAIN
+ - smfiles # Adds a split file's map to the ZCHAIN
- todos PROTO Adds todo list to the ZCHAIN
Read more at the `./MODULES` file
@@ -302,11 +302,6 @@ Public following (uses ak-profile module)
- ak-following # List your follows
- ak-unfollow # Unfollow something
-File splitters
---------------
- - ak-sm-filejoiner # Joins a file from a map
- - ak-sm-filesplitter # Splits a file to chunks and creates a map
-
FS tools
--------
- ak-fs
@@ -336,6 +331,7 @@ Libraries
- lib/_ak_network
- lib/_ak_script
- lib/_ak_log
+ - lib/_ak_smfiles
IPFS Wrappers
-------------
diff --git a/bin/ak-sm-filejoiner b/bin/ak-sm-filejoiner
deleted file mode 100755
index f4165f9..0000000
--- a/bin/ak-sm-filejoiner
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/bin/bash
-#
-set -xe
-
-PROGRAM="$(basename $0)"
-
-usage(){
- echo "$PROGRAM <hash>"
-}
-
-cdaw(){
- pwd > tmp_holder
-}
-
-cdaw
-
-CURDIR="$(cat tmp_holder)"
-
-TMPWD="/tmp/rjs"
-
-if [ ! -d "$TMPWD" ]
-then
- mkdir -p "$TMPWD"
- if [ $? -ne 0 ]
- then
- echo "Can't create $TMPWD dir"
- exit 1
- fi
-fi
-
-MAPSDIR="$AK_WORKDIR/fmp"
-
-CHKDIR="$AK_WORKDIR/ftr"
-
-cd $CHKDIR
-if [ $? -ne 0 ]
-then
- echo "Can't get dir"
- exit 1
-fi
-
-if [ ! -z $1 ]
-then
- MAPSFILE="$1"
-
- echo '#!/bin/bash' > script
-
- # We create a script to copy all the chunks and rename them to their serialized
- # name produced by split when we splitted the file
- awk '{print "cp '$CHKDIR'/"$1" '$TMPWD'/"$2" "}' $MAPSDIR/$MAPSFILE| grep chk >> script
-
- bash script
-
- if [ $? -ne 0 ]
- then
- echo "Error executing copy script"
- exit 1
- fi
- rm script
-
- cd $TMPWD
-
- echo "$PWD"
-
- # Final name we are going to rename to
- OUTPUT="$(tail -n1 $MAPSDIR/$MAPSFILE | awk '{print $2}')"
-
- echo $OUTPUT
-
- # We grep the MAPSFILE for chk filenames in lines and we print them all in
- # one line so the first `cat` will concatenate all chunks to OUTPUT
- cat $(echo -n $(cat $MAPSDIR/$MAPSFILE|grep chk|awk '{print $2" "}'|tr -d '\n')) > $OUTPUT
-
- # We check if everything is okay
- sha512sum -c $MAPSDIR/$MAPSFILE
- if [ $? -ne 0 ]; then
- _ak_log_error "Error while checking sums"
- exit 1
- fi
-
- mv $OUTPUT $CURDIR
-
- rm -rf "$TMPWD"
-
- rm $CURDIR/tmp_holder
-else
- usage
-fi
diff --git a/bin/ak-sm-filesplitter b/bin/ak-sm-filesplitter
deleted file mode 100755
index 4cbb4e4..0000000
--- a/bin/ak-sm-filesplitter
+++ /dev/null
@@ -1,134 +0,0 @@
-#!/bin/bash
-#
-# The concept is simple
-#
-# 1. For a given file we split in 1MB files inside a temporary directory
-#
-# 2. We then create a map file, containing the resulted files and their sha512sum
-#
-# 3. We move the files to our $CHKDIR named after their checksums
-#
-# We ultimately want to be seeding the file so
-#
-# 4. We append the checksum of the original file with its name into the map file
-#
-# 5. We rename the map file after its checksum and move it to maps directory
-#
-# 6. We are done!
-#
-
-# Uncomment next line if you want to debug
-# set -xe
-PROGRAM="$(basename $0)"
-
-if [ ! -f "$1" ]
-then
- echo "ERROR File not found"
- exit 1
-else
- FILE="$1"
-fi
-
-# The directory where the chunked data will be living at
-CHKDIR="$AK_WORKDIR/ftr/"
-# The directory for the map files so we can reconstruct the file
-FILEMAPSDIR="$AK_WORKDIR/fmp/"
-# A temporary root dir to work on
-TEMPORARYDIR="/tmp/tltmp"
-# A subdir to split the files there
-TECHDIR="/tmp/tltmp/chks/"
-# A pin point to return from where we came from
-CURRENTDIR="$(pwd)/"
-
-# Our snippet for logging debug info
-source $AK_LIBDIR/_ak_log
-
-# Checking directories and create them if necessary
-
-# TECHDIR
-if [ ! -d "$TECHDIR" ]
-then
- mkdir -p "$TECHDIR"
- if [ $? -eq 0 ]
- then
- _ak_log_info "Folder $TECHDIR created!"
- else
- _ak_log_error "Problem occured while creating $TECHDIR"
- echo "ERROR Can't create $TECHDIR"
- exit 1
- fi
-else
- _ak_log_info "Temp dir found"
-fi
-
-# FILEMAPSDIR
-if [ ! -d "$FILEMAPSDIR" ]
-then
- mkdir -p "$FILEMAPSDIR"
- if [ $? -eq 0 ]
- then
- _ak_log_info "Folder $FILEMAPSDIR created!"
- else
- _ak_log_error "Problem occured while creating $FILEMAPSDIR"
- echo "ERROR Can't create $FILEMAPSDIR"
- exit 1
- fi
-else
- _ak_log_info "Mapsdir found"
-fi
-
-# CHKDIR
-if [ ! -d "$CHKDIR" ]
-then
- mkdir -p "$CHKDIR"
- if [ $? -eq 0 ]
- then
- _ak_log_info "Folder $CHKDIR created!"
- else
- _ak_log_error "Problem occured while creating $CHKDIR"
- echo "ERROR Can't create $CHKDIR"
- exit 1
- fi
-else
- _ak_log_info "Workdir found"
-fi
-
-# Uncomment next line in case you want to debug the resulting script as well
-# echo 'set -xe' > $TEMPORARYDIR/cmd_queue.sh
-
-# We get the SHA512 hash for the $FILE given
-CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}')
-
-# We split the file into 1048576 bytes and output the chunks into TECHDIR
-split -b 1048576 --additional-suffix ".chk" -d "$FILE" "$TECHDIR$(basename "$FILE")-"
-
-# We go over there...
-cd $TECHDIR
-
-# We get every chunks' SHA512 and we craft a script to rename the chunks and
-# move them to CHKDIR
-sha512sum * > $TEMPORARYDIR/map; while IFS="" read -r p || [ -n "$p" ]
-do
- echo $p | awk '{print "mv " $2 " '$CHKDIR'" $1}' >> $TEMPORARYDIR/cmd_queue.sh
-done < $TEMPORARYDIR/map
-
-# We run the crafted script
-sh $TEMPORARYDIR/cmd_queue.sh
-
-# and we delete it
-rm $TEMPORARYDIR/cmd_queue.sh
-
-# We inform the map about the original $FILE name and SHA512
-echo "$CHECKSUM $(basename "$FILE")" >> $TEMPORARYDIR/map
-
-# We get the SHA512 hash of the resulted map file
-MAPFILEHASH="$(sha512sum $TEMPORARYDIR/map | awk '{ print $1 }')"
-
-# and we rename it with it and move it to FILEMAPSDIR
-`sha512sum $TEMPORARYDIR/map | awk '{print "mv " $2 " '$FILEMAPSDIR'" $1}'`
-
-# We remove the TEMPORARYDIR
-rm -rf $TEMPORARYDIR
-
-# and print the MAPFILEHASH
-echo "$MAPFILEHASH"
diff --git a/lib/_ak_smfiles b/lib/_ak_smfiles
new file mode 100755
index 0000000..55ec381
--- /dev/null
+++ b/lib/_ak_smfiles
@@ -0,0 +1,206 @@
+#!/bin/bash
+source $AK_LIBDIR/_ak_log
+
+# The directory where the chunked data will be living at
+CHKDIR="$AK_WORKDIR/ftr"
+# The directory for the map files so we can reconstruct the file
+FILEMAPSDIR="$AK_WORKDIR/fmp"
+# A temporary root dir to work on
+TEMPORARYDIR="/tmp/tltmp"
+# A subdir to split the files there
+TECHDIR="/tmp/tltmp/chks"
+# A pin point to return from where we came from
+CURRENTDIR="$(pwd)/"
+# Checking directories and create them if necessary
+
+if [ ! -d "$TECHDIR" ]
+then
+ mkdir -p "$TECHDIR"
+ if [ $? -eq 0 ]
+ then
+ _ak_log_info "Folder $TECHDIR created!"
+ else
+ _ak_log_error "Problem occured while creating $TECHDIR"
+ exit 1
+ fi
+else
+ _ak_log_info "Temp dir found"
+fi
+
+if [ ! -d "$FILEMAPSDIR" ]
+then
+ mkdir -p "$FILEMAPSDIR"
+ if [ $? -eq 0 ]
+ then
+ _ak_log_info "Folder $FILEMAPSDIR created!"
+ else
+ _ak_log_error "Problem occured while creating $FILEMAPSDIR"
+ exit 1
+ fi
+else
+ _ak_log_info "Mapsdir found"
+fi
+
+if [ ! -d "$CHKDIR" ]
+then
+ mkdir -p "$CHKDIR"
+ if [ $? -eq 0 ]
+ then
+ _ak_log_info "Folder $CHKDIR created!"
+ else
+ _ak_log_error "Problem occured while creating $CHKDIR"
+ exit 1
+ fi
+else
+ _ak_log_info "Workdir found"
+fi
+
+cdaw(){
+ pwd > tmp_holder
+}
+
+_ak_sm_file_joiner(){
+ cdaw
+
+ CURDIR="$(cat tmp_holder)"
+
+ TMPWD="/tmp/rjs"
+
+ if [ ! -d "$TMPWD" ]
+ then
+ mkdir -p "$TMPWD"
+ if [ $? -ne 0 ]
+ then
+ _ak_log_error "Can't create $TMPWD dir"
+ exit 1
+ fi
+ fi
+
+ cd $CHKDIR
+ if [ $? -ne 0 ]
+ then
+ _ak_log_error "Can't get dir"
+ exit 1
+ fi
+
+ if [ ! -z $1 ]
+ then
+ MAPSFILE="$1"
+
+ echo '#!/bin/bash' > script
+
+ # We create a script to copy all the chunks and rename them to their serialized
+ # name produced by split when we splitted the file
+ awk '{print "cp '$CHKDIR'/"$1" '$TMPWD'/"$2" "}' $FILEMAPSDIR/$MAPSFILE| grep chk >> script
+
+ bash script
+
+ if [ $? -ne 0 ]
+ then
+ echo "Error executing copy script"
+ exit 1
+ fi
+ rm script
+
+ cd $TMPWD
+
+ echo "$PWD"
+
+ # Final name we are going to rename to
+ OUTPUT="$(tail -n 1 $FILEMAPSDIR/$MAPSFILE | awk '{print $2}')"
+
+ echo $OUTPUT
+
+ # We grep the MAPSFILE for chk filenames in lines and we print them all in
+ # one line so the first `cat` will concatenate all chunks to OUTPUT
+ cat $(echo -n $(cat $FILEMAPSDIR/$MAPSFILE|grep chk|awk '{print $2" "}'|tr -d '\n')) > $OUTPUT
+
+ # We check if everything is okay
+ sha512sum -c $FILEMAPSDIR/$MAPSFILE
+ if [ $? -ne 0 ]
+ then
+ _ak_log_error "Error while checking sums"
+ exit 1
+ fi
+
+ mv $OUTPUT $CURDIR
+
+ rm -rf "$TMPWD"
+
+ rm $CURDIR/tmp_holder
+ else
+ _ak_log_error "No hash given"
+ exit 1
+ fi
+}
+
+_ak_sm_file_splitter(){
+ #
+ # The concept is simple
+ #
+ # 1. For a given file we split in 1MB files inside a temporary directory
+ #
+ # 2. We then create a map file, containing the resulted files and their sha512sum
+ #
+ # 3. We move the files to our $CHKDIR named after their checksums
+ #
+ # We ultimately want to be seeding the file so
+ #
+ # 4. We append the checksum of the original file with its name into the map file
+ #
+ # 5. We rename the map file after its checksum and move it to maps directory
+ #
+ # 6. We are done!
+ #
+
+ # Uncomment next line if you want to debug
+ # set -xe
+
+ if [ ! -f "$1" ]
+ then
+ echo "ERROR File not found"
+ exit 1
+ else
+ FILE="$1"
+ fi
+
+ # Uncomment next line in case you want to debug the resulting script as well
+ # echo 'set -xe' > $TEMPORARYDIR/cmd_queue.sh
+
+ # We get the SHA512 hash for the $FILE given
+ CHECKSUM=$(sha512sum "$FILE"|awk '{print $1}')
+
+ # We split the file into 1048576 bytes and output the chunks into TECHDIR
+ split -b 1048576 --additional-suffix ".chk" -d "$FILE" "$TECHDIR/$(basename "$FILE")-"
+
+ # We go over there...
+ cd $TECHDIR
+
+ # We get every chunks' SHA512 and we craft a script to rename the chunks and
+ # move them to CHKDIR
+ sha512sum * > $TEMPORARYDIR/map; while IFS="" read -r p || [ -n "$p" ]
+ do
+ echo $p | awk '{print "mv " $2 " '$CHKDIR'/" $1}' >> $TEMPORARYDIR/cmd_queue.sh
+ done < $TEMPORARYDIR/map
+
+ # We run the crafted script
+ sh $TEMPORARYDIR/cmd_queue.sh
+
+ # and we delete it
+ rm $TEMPORARYDIR/cmd_queue.sh
+
+ # We inform the map about the original $FILE name and SHA512
+ echo "$CHECKSUM $(basename "$FILE")" >> $TEMPORARYDIR/map
+
+ # We get the SHA512 hash of the resulted map file
+ MAPFILEHASH="$(sha512sum $TEMPORARYDIR/map | awk '{ print $1 }')"
+
+ # and we rename it with it and move it to FILEMAPSDIR
+ `sha512sum $TEMPORARYDIR/map | awk '{print "mv " $2 " '$FILEMAPSDIR'/" $1}'`
+
+ # We remove the TEMPORARYDIR
+ rm -rf $TEMPORARYDIR
+
+ # and print the MAPFILEHASH
+ echo "$MAPFILEHASH"
+}
diff --git a/bin/ak-sm-files b/modules/smfiles/lib.sh
index 317149e..ef88278 100755
--- a/bin/ak-sm-files
+++ b/modules/smfiles/lib.sh
@@ -1,41 +1,10 @@
#!/bin/bash
-## sm files
-##
-## -h, --help Prints this help message
-##
-## --add <file> Adds file to zchain as a zblock
-##
-## --index List files
-##
-## --full-index List all files
-##
-## --ls-map-files List map files
-##
-ZFILESDIR="$AK_WORKDIR/files"
-pwd > .pwd
-CRD=$(cat .pwd)
-
-PROGRAM="$(basename $0)"
-#set -xe
+
source $AK_LIBDIR/_ak_log
source $AK_LIBDIR/_ak_ipfs
source $AK_LIBDIR/_ak_gpg
source $AK_LIBDIR/_ak_zblock
-
-if [ ! -d $ZFILESDIR ]; then
- mkdir $ZFILESDIR
- if [ $? == 0 ]
- then
- _ak_log_info "Folder $ZFILESDIR created!"
- else
- _ak_log_error "Failed to create $ZFILESDIR folder"
- exit 1
- fi
- cd $ZFILESDIR
-else
- _ak_log_info "$ZFILESDIR found!"
-fi
-
+source $AK_LIBDIR/_ak_smfiles
_ak_sm_files_add(){
FILENAME="$1"
@@ -51,7 +20,8 @@ _ak_sm_files_main(){
echo "Adding $FILENAME"
_ak_log_info "Switching to tmp folder..."
- if [ $? == 0 ]; then
+ if [ $? -eq 0 ]
+ then
_ak_log_info "Success"
else
_ak_log_error "Error with tmp folder"
@@ -60,7 +30,8 @@ _ak_sm_files_main(){
_ak_log_info "Copying $1 to $TEMPASSIN"
cp $CRP/$FILENAME $FILENAME
- if [ $? == 0 ]; then
+ if [ $? -eq 0 ]
+ then
_ak_log_info "Copied successfully"
else
_ak_log_error "Error copying..."
@@ -68,15 +39,17 @@ _ak_sm_files_main(){
_ak_log_info "Adding $FILENAME to IPFS..."
FILE_IPFS_HASH=$(_ak_ipfs_add $FILENAME)
- if [ $? == 0 ]; then
+ if [ $? -eq 0 ]
+ then
_ak_log_info "Added $FILENAME to IPFS"
else
_ak_log_error "Error in adding the $FILENAME to IPFS"
fi
_ak_log_info "Adding $FILE to SHAMAPSYS..."
- FILEMAP_SHA512_HASH=$(ak-sm-filesplitter $FILENAME)
- if [ $? == 0 ]; then
+ FILEMAP_SHA512_HASH=$(_ak_sm_file_splitter $FILENAME)
+ if [ $? -eq 0 ]
+ then
_ak_log_info "Added $FILENAME to SHAMAPSYS"
else
_ak_log_error "Error in adding the $FILENAME to SHAMAPSYS"
@@ -85,7 +58,8 @@ _ak_sm_files_main(){
_ak_log_info "Signing..."
SIGN_FILE=$FILENAME".asc"
_ak_gpg_sign_detached $SIGN_FILE $FILENAME
- if [ $? == 0 ]; then
+ if [ $? -eq 0 ]
+ then
_ak_log_info "Signed"
else
_ak_log_error "Error while signing"
@@ -93,15 +67,17 @@ _ak_sm_files_main(){
_ak_log_info "Adding signature to IPFS"
SIGNATURE=$(_ak_ipfs_add $SIGN_FILE)
- if [ $? == 0 ]; then
+ if [ $? -eq 0 ]
+ then
_ak_log_info "Added"
else
_ak_log_error "Error while adding"
fi
_ak_log_info "Adding signature to SHAMAPSYS"
- SHAMAPSIGMAP=$(ak-sm-filesplitter $SIGN_FILE)
- if [ $? == 0 ]; then
+ SHAMAPSIGMAP=$(_ak_sm_file_splitter $SIGN_FILE)
+ if [ $? -eq 0 ]
+ then
_ak_log_info "Added"
else
_ak_log_error "Error while adding"
@@ -123,7 +99,7 @@ EOF
echo "Publishing..."
_ak_zblock_pack sha-files/announce $(pwd)/data
- if [ $? == 0 ]
+ if [ $? -eq 0 ]
then
echo "cool"
else
@@ -138,26 +114,16 @@ _ak_sm_files_index(){
_ak_sm_files_ls_mapfiles(){
cd $AK_WORKDIR/fmp
- for f in `find . -type f | sed -e 's/\.\///g'`;do
- FILENAME="$(tail -n1 $f | grep '^[abcdef1234567890]' | awk '{ print $2 }')"
- FILEHASH="$(tail -n1 $f | grep '^[abcdef1234567890]' | awk '{ print $1 }')"
+ for f in `find . -type f | sed -e 's/\.\///g'`
+ do
+ FILENAME="$(tail -n 1 $f | grep '^[abcdef1234567890]' | awk '{ print $2 }')"
+ FILEHASH="$(tail -n 1 $f | grep '^[abcdef1234567890]' | awk '{ print $1 }')"
MAPFILE="$f"
printf "\nMap: %s\nFilename: %s\nSum: %s\n\n" $MAPFILE $FILENAME $FILEHASH
done
}
_ak_sm_files_full_index(){
- tail -n1 $AK_WORKDIR/fmp/* | grep '^[abcdef1234567890]'
+ tail -n 1 $AK_WORKDIR/fmp/* | grep '^[abcdef1234567890]'
}
-if [ ! -z $1 ]; then
- case $1 in
- -h | --help) usage; exit;;
- --add) _ak_sm_files_add $2; exit;;
- --index) _ak_sm_files_index; exit;;
- --full-index) _ak_sm_files_full_index; exit;;
- --ls-map-files) _ak_sm_files_ls_mapfiles; exit;;
- *) usage; exit;;
- esac
-else usage
-fi
diff --git a/modules/smfiles/main.sh b/modules/smfiles/main.sh
new file mode 100755
index 0000000..f7662c3
--- /dev/null
+++ b/modules/smfiles/main.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+## sm files
+##
+## -h, --help Prints this help message
+##
+## --add <file> Adds file to zchain as a zblock
+##
+## --index List files
+##
+## --full-index List all files
+##
+## --ls-map-files List map files
+##
+ZFILESDIR="$AK_WORKDIR/files"
+pwd > .pwd
+CRD=$(cat .pwd)
+
+fullprogrampath="$(realpath $0)"
+PROGRAM="$(realpath $0 | rev |cut -d '/' -f 2 | rev)"
+descriptionString="Module to files in zchain"
+
+source $AK_LIBDIR/_ak_log
+source $AK_LIBDIR/_ak_script
+source $AK_MODULESDIR/$PROGRAM/lib.sh
+
+if [ ! -d $ZFILESDIR ]
+then
+ mkdir $ZFILESDIR
+ if [ $? -eq 0 ]
+ then
+ _ak_log_info "Folder $ZFILESDIR created!"
+ else
+ _ak_log_error "Failed to create $ZFILESDIR folder"
+ exit 1
+ fi
+ cd $ZFILESDIR
+else
+ _ak_log_info "$ZFILESDIR found!"
+fi
+
+
+
+if [ ! -z $1 ]
+then
+ case $1 in
+ -h | --help) _ak_usage; exit;;
+ --add) _ak_sm_files_add $2; exit;;
+ --index) _ak_sm_files_index; exit;;
+ --full-index) _ak_sm_files_full_index; exit;;
+ --ls-map-files) _ak_sm_files_ls_mapfiles; exit;;
+ *) _ak_usage; exit;;
+ esac
+else
+ _ak_usage
+fi
+