mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-15 16:30:55 +01:00
7f4fb34182
It is better developing vmctl tool in VictoriaMetrics repository, so it could be released together with the rest of vmutils tools such as vmalert, vmagent, vmbackup, vmrestore and vmauth.
414 lines
12 KiB
Bash
414 lines
12 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Purpose: plain text tar format
|
|
# Limitations: - only suitable for text files, directories, and symlinks
|
|
# - stores only filename, content, and mode
|
|
# - not designed for untrusted input
|
|
#
|
|
# Note: must work with bash version 3.2 (macOS)
|
|
|
|
# Copyright 2017 Roger Luethi
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
set -o errexit -o nounset
|
|
|
|
# Sanitize environment (for instance, standard sorting of glob matches)
|
|
export LC_ALL=C
|
|
|
|
path=""
|
|
CMD=""
|
|
ARG_STRING="$*"
|
|
|
|
#------------------------------------------------------------------------------
|
|
# Not all sed implementations can work on null bytes. In order to make ttar
|
|
# work out of the box on macOS, use Python as a stream editor.
|
|
|
|
USE_PYTHON=0
|
|
|
|
PYTHON_CREATE_FILTER=$(cat << 'PCF'
|
|
#!/usr/bin/env python
|
|
|
|
import re
|
|
import sys
|
|
|
|
for line in sys.stdin:
|
|
line = re.sub(r'EOF', r'\EOF', line)
|
|
line = re.sub(r'NULLBYTE', r'\NULLBYTE', line)
|
|
line = re.sub('\x00', r'NULLBYTE', line)
|
|
sys.stdout.write(line)
|
|
PCF
|
|
)
|
|
|
|
PYTHON_EXTRACT_FILTER=$(cat << 'PEF'
|
|
#!/usr/bin/env python
|
|
|
|
import re
|
|
import sys
|
|
|
|
for line in sys.stdin:
|
|
line = re.sub(r'(?<!\\)NULLBYTE', '\x00', line)
|
|
line = re.sub(r'\\NULLBYTE', 'NULLBYTE', line)
|
|
line = re.sub(r'([^\\])EOF', r'\1', line)
|
|
line = re.sub(r'\\EOF', 'EOF', line)
|
|
sys.stdout.write(line)
|
|
PEF
|
|
)
|
|
|
|
function test_environment {
|
|
if [[ "$(echo "a" | sed 's/a/\x0/' | wc -c)" -ne 2 ]]; then
|
|
echo "WARNING sed unable to handle null bytes, using Python (slow)."
|
|
if ! which python >/dev/null; then
|
|
echo "ERROR Python not found. Aborting."
|
|
exit 2
|
|
fi
|
|
USE_PYTHON=1
|
|
fi
|
|
}
|
|
|
|
#------------------------------------------------------------------------------
|
|
|
|
function usage {
|
|
bname=$(basename "$0")
|
|
cat << USAGE
|
|
Usage: $bname [-C <DIR>] -c -f <ARCHIVE> <FILE...> (create archive)
|
|
$bname -t -f <ARCHIVE> (list archive contents)
|
|
$bname [-C <DIR>] -x -f <ARCHIVE> (extract archive)
|
|
|
|
Options:
|
|
-C <DIR> (change directory)
|
|
-v (verbose)
|
|
--recursive-unlink (recursively delete existing directory if path
|
|
collides with file or directory to extract)
|
|
|
|
Example: Change to sysfs directory, create ttar file from fixtures directory
|
|
$bname -C sysfs -c -f sysfs/fixtures.ttar fixtures/
|
|
USAGE
|
|
exit "$1"
|
|
}
|
|
|
|
function vecho {
|
|
if [ "${VERBOSE:-}" == "yes" ]; then
|
|
echo >&7 "$@"
|
|
fi
|
|
}
|
|
|
|
function set_cmd {
|
|
if [ -n "$CMD" ]; then
|
|
echo "ERROR: more than one command given"
|
|
echo
|
|
usage 2
|
|
fi
|
|
CMD=$1
|
|
}
|
|
|
|
unset VERBOSE
|
|
unset RECURSIVE_UNLINK
|
|
|
|
while getopts :cf:-:htxvC: opt; do
|
|
case $opt in
|
|
c)
|
|
set_cmd "create"
|
|
;;
|
|
f)
|
|
ARCHIVE=$OPTARG
|
|
;;
|
|
h)
|
|
usage 0
|
|
;;
|
|
t)
|
|
set_cmd "list"
|
|
;;
|
|
x)
|
|
set_cmd "extract"
|
|
;;
|
|
v)
|
|
VERBOSE=yes
|
|
exec 7>&1
|
|
;;
|
|
C)
|
|
CDIR=$OPTARG
|
|
;;
|
|
-)
|
|
case $OPTARG in
|
|
recursive-unlink)
|
|
RECURSIVE_UNLINK="yes"
|
|
;;
|
|
*)
|
|
echo -e "Error: invalid option -$OPTARG"
|
|
echo
|
|
usage 1
|
|
;;
|
|
esac
|
|
;;
|
|
*)
|
|
echo >&2 "ERROR: invalid option -$OPTARG"
|
|
echo
|
|
usage 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Remove processed options from arguments
|
|
shift $(( OPTIND - 1 ));
|
|
|
|
if [ "${CMD:-}" == "" ]; then
|
|
echo >&2 "ERROR: no command given"
|
|
echo
|
|
usage 1
|
|
elif [ "${ARCHIVE:-}" == "" ]; then
|
|
echo >&2 "ERROR: no archive name given"
|
|
echo
|
|
usage 1
|
|
fi
|
|
|
|
function list {
|
|
local path=""
|
|
local size=0
|
|
local line_no=0
|
|
local ttar_file=$1
|
|
if [ -n "${2:-}" ]; then
|
|
echo >&2 "ERROR: too many arguments."
|
|
echo
|
|
usage 1
|
|
fi
|
|
if [ ! -e "$ttar_file" ]; then
|
|
echo >&2 "ERROR: file not found ($ttar_file)"
|
|
echo
|
|
usage 1
|
|
fi
|
|
while read -r line; do
|
|
line_no=$(( line_no + 1 ))
|
|
if [ $size -gt 0 ]; then
|
|
size=$(( size - 1 ))
|
|
continue
|
|
fi
|
|
if [[ $line =~ ^Path:\ (.*)$ ]]; then
|
|
path=${BASH_REMATCH[1]}
|
|
elif [[ $line =~ ^Lines:\ (.*)$ ]]; then
|
|
size=${BASH_REMATCH[1]}
|
|
echo "$path"
|
|
elif [[ $line =~ ^Directory:\ (.*)$ ]]; then
|
|
path=${BASH_REMATCH[1]}
|
|
echo "$path/"
|
|
elif [[ $line =~ ^SymlinkTo:\ (.*)$ ]]; then
|
|
echo "$path -> ${BASH_REMATCH[1]}"
|
|
fi
|
|
done < "$ttar_file"
|
|
}
|
|
|
|
function extract {
|
|
local path=""
|
|
local size=0
|
|
local line_no=0
|
|
local ttar_file=$1
|
|
if [ -n "${2:-}" ]; then
|
|
echo >&2 "ERROR: too many arguments."
|
|
echo
|
|
usage 1
|
|
fi
|
|
if [ ! -e "$ttar_file" ]; then
|
|
echo >&2 "ERROR: file not found ($ttar_file)"
|
|
echo
|
|
usage 1
|
|
fi
|
|
while IFS= read -r line; do
|
|
line_no=$(( line_no + 1 ))
|
|
local eof_without_newline
|
|
if [ "$size" -gt 0 ]; then
|
|
if [[ "$line" =~ [^\\]EOF ]]; then
|
|
# An EOF not preceded by a backslash indicates that the line
|
|
# does not end with a newline
|
|
eof_without_newline=1
|
|
else
|
|
eof_without_newline=0
|
|
fi
|
|
# Replace NULLBYTE with null byte if at beginning of line
|
|
# Replace NULLBYTE with null byte unless preceded by backslash
|
|
# Remove one backslash in front of NULLBYTE (if any)
|
|
# Remove EOF unless preceded by backslash
|
|
# Remove one backslash in front of EOF
|
|
if [ $USE_PYTHON -eq 1 ]; then
|
|
echo -n "$line" | python -c "$PYTHON_EXTRACT_FILTER" >> "$path"
|
|
else
|
|
# The repeated pattern makes up for sed's lack of negative
|
|
# lookbehind assertions (for consecutive null bytes).
|
|
echo -n "$line" | \
|
|
sed -e 's/^NULLBYTE/\x0/g;
|
|
s/\([^\\]\)NULLBYTE/\1\x0/g;
|
|
s/\([^\\]\)NULLBYTE/\1\x0/g;
|
|
s/\\NULLBYTE/NULLBYTE/g;
|
|
s/\([^\\]\)EOF/\1/g;
|
|
s/\\EOF/EOF/g;
|
|
' >> "$path"
|
|
fi
|
|
if [[ "$eof_without_newline" -eq 0 ]]; then
|
|
echo >> "$path"
|
|
fi
|
|
size=$(( size - 1 ))
|
|
continue
|
|
fi
|
|
if [[ $line =~ ^Path:\ (.*)$ ]]; then
|
|
path=${BASH_REMATCH[1]}
|
|
if [ -L "$path" ]; then
|
|
rm "$path"
|
|
elif [ -d "$path" ]; then
|
|
if [ "${RECURSIVE_UNLINK:-}" == "yes" ]; then
|
|
rm -r "$path"
|
|
else
|
|
# Safe because symlinks to directories are dealt with above
|
|
rmdir "$path"
|
|
fi
|
|
elif [ -e "$path" ]; then
|
|
rm "$path"
|
|
fi
|
|
elif [[ $line =~ ^Lines:\ (.*)$ ]]; then
|
|
size=${BASH_REMATCH[1]}
|
|
# Create file even if it is zero-length.
|
|
touch "$path"
|
|
vecho " $path"
|
|
elif [[ $line =~ ^Mode:\ (.*)$ ]]; then
|
|
mode=${BASH_REMATCH[1]}
|
|
chmod "$mode" "$path"
|
|
vecho "$mode"
|
|
elif [[ $line =~ ^Directory:\ (.*)$ ]]; then
|
|
path=${BASH_REMATCH[1]}
|
|
mkdir -p "$path"
|
|
vecho " $path/"
|
|
elif [[ $line =~ ^SymlinkTo:\ (.*)$ ]]; then
|
|
ln -s "${BASH_REMATCH[1]}" "$path"
|
|
vecho " $path -> ${BASH_REMATCH[1]}"
|
|
elif [[ $line =~ ^# ]]; then
|
|
# Ignore comments between files
|
|
continue
|
|
else
|
|
echo >&2 "ERROR: Unknown keyword on line $line_no: $line"
|
|
exit 1
|
|
fi
|
|
done < "$ttar_file"
|
|
}
|
|
|
|
function div {
|
|
echo "# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -" \
|
|
"- - - - - -"
|
|
}
|
|
|
|
function get_mode {
|
|
local mfile=$1
|
|
if [ -z "${STAT_OPTION:-}" ]; then
|
|
if stat -c '%a' "$mfile" >/dev/null 2>&1; then
|
|
# GNU stat
|
|
STAT_OPTION='-c'
|
|
STAT_FORMAT='%a'
|
|
else
|
|
# BSD stat
|
|
STAT_OPTION='-f'
|
|
# Octal output, user/group/other (omit file type, sticky bit)
|
|
STAT_FORMAT='%OLp'
|
|
fi
|
|
fi
|
|
stat "${STAT_OPTION}" "${STAT_FORMAT}" "$mfile"
|
|
}
|
|
|
|
function _create {
|
|
shopt -s nullglob
|
|
local mode
|
|
local eof_without_newline
|
|
while (( "$#" )); do
|
|
file=$1
|
|
if [ -L "$file" ]; then
|
|
echo "Path: $file"
|
|
symlinkTo=$(readlink "$file")
|
|
echo "SymlinkTo: $symlinkTo"
|
|
vecho " $file -> $symlinkTo"
|
|
div
|
|
elif [ -d "$file" ]; then
|
|
# Strip trailing slash (if there is one)
|
|
file=${file%/}
|
|
echo "Directory: $file"
|
|
mode=$(get_mode "$file")
|
|
echo "Mode: $mode"
|
|
vecho "$mode $file/"
|
|
div
|
|
# Find all files and dirs, including hidden/dot files
|
|
for x in "$file/"{*,.[^.]*}; do
|
|
_create "$x"
|
|
done
|
|
elif [ -f "$file" ]; then
|
|
echo "Path: $file"
|
|
lines=$(wc -l "$file"|awk '{print $1}')
|
|
eof_without_newline=0
|
|
if [[ "$(wc -c "$file"|awk '{print $1}')" -gt 0 ]] && \
|
|
[[ "$(tail -c 1 "$file" | wc -l)" -eq 0 ]]; then
|
|
eof_without_newline=1
|
|
lines=$((lines+1))
|
|
fi
|
|
echo "Lines: $lines"
|
|
# Add backslash in front of EOF
|
|
# Add backslash in front of NULLBYTE
|
|
# Replace null byte with NULLBYTE
|
|
if [ $USE_PYTHON -eq 1 ]; then
|
|
< "$file" python -c "$PYTHON_CREATE_FILTER"
|
|
else
|
|
< "$file" \
|
|
sed 's/EOF/\\EOF/g;
|
|
s/NULLBYTE/\\NULLBYTE/g;
|
|
s/\x0/NULLBYTE/g;
|
|
'
|
|
fi
|
|
if [[ "$eof_without_newline" -eq 1 ]]; then
|
|
# Finish line with EOF to indicate that the original line did
|
|
# not end with a linefeed
|
|
echo "EOF"
|
|
fi
|
|
mode=$(get_mode "$file")
|
|
echo "Mode: $mode"
|
|
vecho "$mode $file"
|
|
div
|
|
else
|
|
echo >&2 "ERROR: file not found ($file in $(pwd))"
|
|
exit 2
|
|
fi
|
|
shift
|
|
done
|
|
}
|
|
|
|
function create {
|
|
ttar_file=$1
|
|
shift
|
|
if [ -z "${1:-}" ]; then
|
|
echo >&2 "ERROR: missing arguments."
|
|
echo
|
|
usage 1
|
|
fi
|
|
if [ -e "$ttar_file" ]; then
|
|
rm "$ttar_file"
|
|
fi
|
|
exec > "$ttar_file"
|
|
echo "# Archive created by ttar $ARG_STRING"
|
|
_create "$@"
|
|
}
|
|
|
|
test_environment
|
|
|
|
if [ -n "${CDIR:-}" ]; then
|
|
if [[ "$ARCHIVE" != /* ]]; then
|
|
# Relative path: preserve the archive's location before changing
|
|
# directory
|
|
ARCHIVE="$(pwd)/$ARCHIVE"
|
|
fi
|
|
cd "$CDIR"
|
|
fi
|
|
|
|
"$CMD" "$ARCHIVE" "$@"
|