From d80d72efec4d6aaab3ce685376d66bc08ea9933f Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 29 Sep 2021 12:10:26 +0300 Subject: [PATCH] app/{vmbackup,vmrestore}: switch from `gcs://...` to `gs://...` urls for backups to GCS The `gs://` urls are commonly used, so prefer them instead of `gcs://` urls, while leaving support for `gcs://` urls for backwards compatibility. --- app/vmbackup/README.md | 16 ++++++++-------- app/vmbackup/main.go | 2 +- app/vmbackupmanager/README.md | 4 ++-- app/vmrestore/README.md | 4 ++-- app/vmrestore/main.go | 2 +- docs/vmbackup.md | 16 ++++++++-------- docs/vmbackupmanager.md | 4 ++-- docs/vmrestore.md | 4 ++-- lib/backup/actions/util.go | 4 ++-- 9 files changed, 28 insertions(+), 28 deletions(-) diff --git a/app/vmbackup/README.md b/app/vmbackup/README.md index 81439639d..b872ee10c 100644 --- a/app/vmbackup/README.md +++ b/app/vmbackup/README.md @@ -4,7 +4,7 @@ Supported storage systems for backups: -* [GCS](https://cloud.google.com/storage/). Example: `gcs:///` +* [GCS](https://cloud.google.com/storage/). Example: `gs:///` * [S3](https://aws.amazon.com/s3/). Example: `s3:///` * Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details. * Local filesystem. Example: `fs://` @@ -30,7 +30,7 @@ creation of hourly, daily, weekly and monthly backups. Regular backup can be performed with the following command: ``` -vmbackup -storageDataPath= -snapshotName= -dst=gcs:/// +vmbackup -storageDataPath= -snapshotName= -dst=gs:/// ``` * `` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`. @@ -46,7 +46,7 @@ If the destination GCS bucket already contains the previous backup at `-origin` with the following command: ``` -vmbackup -storageDataPath= -snapshotName= -dst=gcs:/// -origin=gcs:/// +vmbackup -storageDataPath= -snapshotName= -dst=gs:/// -origin=gs:/// ``` It saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`. @@ -58,7 +58,7 @@ Incremental backups performed if `-dst` points to an already existing backup. In It saves time and network bandwidth costs when working with big backups: ``` -vmbackup -storageDataPath= -snapshotName= -dst=gcs:/// +vmbackup -storageDataPath= -snapshotName= -dst=gs:/// ``` @@ -69,16 +69,16 @@ Smart backups mean storing full daily backups into `YYYYMMDD` folders and creati * Run the following command every hour: ``` -vmbackup -snapshotName= -dst=gcs:///latest +vmbackup -snapshotName= -dst=gs:///latest ``` Where `` is the latest [snapshot](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots). -The command will upload only changed data to `gcs:///latest`. +The command will upload only changed data to `gs:///latest`. * Run the following command once a day: ``` -vmbackup -snapshotName= -dst=gcs:/// -origin=gcs:///latest +vmbackup -snapshotName= -dst=gs:/// -origin=gs:///latest ``` Where `` is the snapshot for the last day ``. @@ -183,7 +183,7 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time- -customS3Endpoint string Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set -dst string - Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir + Where to put the backup on the remote storage. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir -dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded -envflag.enable Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details diff --git a/app/vmbackup/main.go b/app/vmbackup/main.go index 3be0ea54a..c90aeb0a8 100644 --- a/app/vmbackup/main.go +++ b/app/vmbackup/main.go @@ -25,7 +25,7 @@ var ( snapshotDeleteURL = flag.String("snapshot.deleteURL", "", "VictoriaMetrics delete snapshot url. Optional. Will be generated from -snapshot.createURL if not provided. "+ "All created snapshots will be automatically deleted. Example: http://victoriametrics:8428/snapshot/delete") dst = flag.String("dst", "", "Where to put the backup on the remote storage. "+ - "Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+ + "Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir\n"+ "-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded") origin = flag.String("origin", "", "Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups") concurrency = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce backup duration") diff --git a/app/vmbackupmanager/README.md b/app/vmbackupmanager/README.md index 7e9ecc732..111d4cc9b 100644 --- a/app/vmbackupmanager/README.md +++ b/app/vmbackupmanager/README.md @@ -76,7 +76,7 @@ Backup manager launched with the following configuration: ```console export NODE_IP=192.168.0.10 export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428 -./vmbackupmanager -dst=gcs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula +./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula ``` Expected logs in vmbackupmanager: @@ -125,7 +125,7 @@ We enable backup retention policy for backup manager by using following configur ```console export NODE_IP=192.168.0.10 export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428 -./vmbackupmanager -dst=gcs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create +./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -keepLastDaily=3 -eula ``` diff --git a/app/vmrestore/README.md b/app/vmrestore/README.md index e1c5b4718..8eaefd2e1 100644 --- a/app/vmrestore/README.md +++ b/app/vmrestore/README.md @@ -12,7 +12,7 @@ when restarting `vmrestore` with the same args. VictoriaMetrics must be stopped during the restore process. ``` -vmrestore -src=gcs:/// -storageDataPath= +vmrestore -src=gs:/// -storageDataPath= ``` @@ -116,7 +116,7 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q -skipBackupCompleteCheck Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file -src string - Source path with backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir + Source path with backup on the remote storage. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir -storageDataPath string Destination path where backup must be restored. VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case the contents of -storageDataPath dir is synchronized with -src contents, i.e. it works like 'rsync --delete' (default "victoria-metrics-data") -version diff --git a/app/vmrestore/main.go b/app/vmrestore/main.go index ac5de3d8f..9db012846 100644 --- a/app/vmrestore/main.go +++ b/app/vmrestore/main.go @@ -16,7 +16,7 @@ import ( var ( src = flag.String("src", "", "Source path with backup on the remote storage. "+ - "Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir") + "Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir") storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Destination path where backup must be restored. "+ "VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case the contents of -storageDataPath dir "+ "is synchronized with -src contents, i.e. it works like 'rsync --delete'") diff --git a/docs/vmbackup.md b/docs/vmbackup.md index cd7311d22..cc1ea6a4a 100644 --- a/docs/vmbackup.md +++ b/docs/vmbackup.md @@ -8,7 +8,7 @@ sort: 6 Supported storage systems for backups: -* [GCS](https://cloud.google.com/storage/). Example: `gcs:///` +* [GCS](https://cloud.google.com/storage/). Example: `gs:///` * [S3](https://aws.amazon.com/s3/). Example: `s3:///` * Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/docs/mimic/radosgw/s3/) or [Swift](https://www.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details. * Local filesystem. Example: `fs://` @@ -34,7 +34,7 @@ creation of hourly, daily, weekly and monthly backups. Regular backup can be performed with the following command: ``` -vmbackup -storageDataPath= -snapshotName= -dst=gcs:/// +vmbackup -storageDataPath= -snapshotName= -dst=gs:/// ``` * `` - path to VictoriaMetrics data pointed by `-storageDataPath` command-line flag in single-node VictoriaMetrics or in cluster `vmstorage`. @@ -50,7 +50,7 @@ If the destination GCS bucket already contains the previous backup at `-origin` with the following command: ``` -vmbackup -storageDataPath= -snapshotName= -dst=gcs:/// -origin=gcs:/// +vmbackup -storageDataPath= -snapshotName= -dst=gs:/// -origin=gs:/// ``` It saves time and network bandwidth costs by performing server-side copy for the shared data from the `-origin` to `-dst`. @@ -62,7 +62,7 @@ Incremental backups performed if `-dst` points to an already existing backup. In It saves time and network bandwidth costs when working with big backups: ``` -vmbackup -storageDataPath= -snapshotName= -dst=gcs:/// +vmbackup -storageDataPath= -snapshotName= -dst=gs:/// ``` @@ -73,16 +73,16 @@ Smart backups mean storing full daily backups into `YYYYMMDD` folders and creati * Run the following command every hour: ``` -vmbackup -snapshotName= -dst=gcs:///latest +vmbackup -snapshotName= -dst=gs:///latest ``` Where `` is the latest [snapshot](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-work-with-snapshots). -The command will upload only changed data to `gcs:///latest`. +The command will upload only changed data to `gs:///latest`. * Run the following command once a day: ``` -vmbackup -snapshotName= -dst=gcs:/// -origin=gcs:///latest +vmbackup -snapshotName= -dst=gs:/// -origin=gs:///latest ``` Where `` is the snapshot for the last day ``. @@ -187,7 +187,7 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time- -customS3Endpoint string Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set -dst string - Where to put the backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir + Where to put the backup on the remote storage. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir -dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded -envflag.enable Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set. See https://docs.victoriametrics.com/#environment-variables for more details diff --git a/docs/vmbackupmanager.md b/docs/vmbackupmanager.md index 0ccfcf27c..3eb7bf3fa 100644 --- a/docs/vmbackupmanager.md +++ b/docs/vmbackupmanager.md @@ -80,7 +80,7 @@ Backup manager launched with the following configuration: ```console export NODE_IP=192.168.0.10 export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428 -./vmbackupmanager -dst=gcs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula +./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -eula ``` Expected logs in vmbackupmanager: @@ -129,7 +129,7 @@ We enable backup retention policy for backup manager by using following configur ```console export NODE_IP=192.168.0.10 export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428 -./vmbackupmanager -dst=gcs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create +./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -keepLastDaily=3 -eula ``` diff --git a/docs/vmrestore.md b/docs/vmrestore.md index 66c73be37..e689f2173 100644 --- a/docs/vmrestore.md +++ b/docs/vmrestore.md @@ -16,7 +16,7 @@ when restarting `vmrestore` with the same args. VictoriaMetrics must be stopped during the restore process. ``` -vmrestore -src=gcs:/// -storageDataPath= +vmrestore -src=gs:/// -storageDataPath= ``` @@ -120,7 +120,7 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q -skipBackupCompleteCheck Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file -src string - Source path with backup on the remote storage. Example: gcs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir + Source path with backup on the remote storage. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir -storageDataPath string Destination path where backup must be restored. VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case the contents of -storageDataPath dir is synchronized with -src contents, i.e. it works like 'rsync --delete' (default "victoria-metrics-data") -version diff --git a/lib/backup/actions/util.go b/lib/backup/actions/util.go index 5eebe5b1a..bc80fe57a 100644 --- a/lib/backup/actions/util.go +++ b/lib/backup/actions/util.go @@ -182,7 +182,7 @@ func NewRemoteFS(path string) (common.RemoteFS, error) { } n := strings.Index(path, "://") if n < 0 { - return nil, fmt.Errorf("Missing scheme in path %q. Supported schemes: `gcs://`, `s3://`, `fs://`", path) + return nil, fmt.Errorf("Missing scheme in path %q. Supported schemes: `gs://`, `s3://`, `fs://`", path) } scheme := path[:n] dir := path[n+len("://"):] @@ -195,7 +195,7 @@ func NewRemoteFS(path string) (common.RemoteFS, error) { Dir: dir, } return fs, nil - case "gcs": + case "gcs", "gs": n := strings.Index(dir, "/") if n < 0 { return nil, fmt.Errorf("missing directory on the gcs bucket %q", dir)