feat: add backups feature
This commit is contained in:
3252a8
@@ -0,0 +1,108 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from config.settings import Settings
|
||||
|
||||
BACKUP_APP_ID = "remnawave-minishop"
|
||||
BACKUP_FILENAME_PREFIX = "remnawave-minishop-backup-"
|
||||
BACKUP_FORMAT_VERSION = 1
|
||||
BACKUP_MANIFEST_NAME = "manifest.json"
|
||||
|
||||
|
||||
def backup_signature_secret(settings: Settings) -> str:
|
||||
configured = str(getattr(settings, "BACKUP_ARCHIVE_SIGNATURE_SECRET", "") or "").strip()
|
||||
return configured or settings.BOT_TOKEN
|
||||
|
||||
|
||||
def canonical_manifest_payload(manifest: dict[str, Any]) -> bytes:
|
||||
payload = json.loads(json.dumps(manifest, ensure_ascii=False))
|
||||
archive = payload.get("archive")
|
||||
if isinstance(archive, dict):
|
||||
archive.pop("signature", None)
|
||||
return json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def sign_manifest(manifest: dict[str, Any], settings: Settings) -> str:
|
||||
return hmac.new(
|
||||
backup_signature_secret(settings).encode("utf-8"),
|
||||
canonical_manifest_payload(manifest),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def verify_manifest_signature(manifest: dict[str, Any], settings: Settings) -> bool:
|
||||
archive = manifest.get("archive") if isinstance(manifest.get("archive"), dict) else {}
|
||||
signature = str(archive.get("signature") or "")
|
||||
if not signature:
|
||||
return False
|
||||
expected = sign_manifest(manifest, settings)
|
||||
return hmac.compare_digest(signature, expected)
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def build_file_records(source_dir: Path) -> list[dict[str, Any]]:
|
||||
records: list[dict[str, Any]] = []
|
||||
for path in sorted(source_dir.rglob("*")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
relative = path.relative_to(source_dir).as_posix()
|
||||
if relative == BACKUP_MANIFEST_NAME:
|
||||
continue
|
||||
stat = path.stat()
|
||||
records.append(
|
||||
{
|
||||
"path": relative,
|
||||
"size_bytes": int(stat.st_size),
|
||||
"sha256": file_sha256(path),
|
||||
}
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
def attach_archive_integrity(
|
||||
manifest: dict[str, Any],
|
||||
*,
|
||||
file_records: list[dict[str, Any]],
|
||||
settings: Settings,
|
||||
) -> None:
|
||||
manifest["app"] = BACKUP_APP_ID
|
||||
manifest["format_version"] = BACKUP_FORMAT_VERSION
|
||||
manifest["archive"] = {
|
||||
"files": file_records,
|
||||
}
|
||||
manifest["archive"]["signature"] = sign_manifest(manifest, settings)
|
||||
|
||||
|
||||
def write_manifest(source_dir: Path, manifest: dict[str, Any]) -> None:
|
||||
(source_dir / BACKUP_MANIFEST_NAME).write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def write_zip_from_directory(source_dir: Path, archive_path: Path) -> None:
|
||||
with zipfile.ZipFile(
|
||||
archive_path,
|
||||
mode="w",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
compresslevel=6,
|
||||
) as archive:
|
||||
for path in sorted(source_dir.rglob("*")):
|
||||
if path.is_file():
|
||||
archive.write(path, path.relative_to(source_dir).as_posix())
|
||||
@@ -0,0 +1,660 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Optional
|
||||
|
||||
from bot.services.backup_archive import (
|
||||
BACKUP_APP_ID,
|
||||
BACKUP_FILENAME_PREFIX,
|
||||
BACKUP_FORMAT_VERSION,
|
||||
BACKUP_MANIFEST_NAME,
|
||||
attach_archive_integrity,
|
||||
build_file_records,
|
||||
verify_manifest_signature,
|
||||
write_manifest,
|
||||
write_zip_from_directory,
|
||||
)
|
||||
from bot.services.backup_worker import (
|
||||
DEFAULT_COMPOSE_EXCLUDED_DIRS,
|
||||
)
|
||||
from config.settings import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BACKUP_UPLOAD_MAX_BYTES = 2 * 1024 * 1024 * 1024
|
||||
BACKUP_MAX_MEMBERS = 20_000
|
||||
BACKUP_MAX_MEMBER_BYTES = 4 * 1024 * 1024 * 1024
|
||||
BACKUP_MAX_UNCOMPRESSED_BYTES = 16 * 1024 * 1024 * 1024
|
||||
BACKUP_MAX_COMPOSE_BYTES = 1024 * 1024 * 1024
|
||||
BACKUP_MAX_COMPOSE_MEMBER_BYTES = 256 * 1024 * 1024
|
||||
BACKUP_MAX_COMPRESSION_RATIO = 200
|
||||
BACKUP_ZIP_BOMB_MIN_BYTES = 100 * 1024 * 1024
|
||||
COMPOSE_PRE_RESTORE_PREFIX = "remnawave-minishop-compose-pre-restore-"
|
||||
SAFE_ARCHIVE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.@+-]{0,220}\.zip$")
|
||||
|
||||
|
||||
class BackupArchiveError(ValueError):
|
||||
"""The selected archive cannot be used for restore."""
|
||||
|
||||
|
||||
class BackupRestoreError(RuntimeError):
|
||||
"""Restore command failed after archive validation."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackupArchiveInfo:
|
||||
name: str
|
||||
path: Path
|
||||
size_bytes: int
|
||||
modified_at: datetime
|
||||
created_at: Optional[str] = None
|
||||
created_at_local: Optional[str] = None
|
||||
has_database: bool = False
|
||||
has_compose: bool = False
|
||||
database_name: Optional[str] = None
|
||||
compose_files_count: int = 0
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
manifest: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_payload(self) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"size_bytes": self.size_bytes,
|
||||
"modified_at": self.modified_at.isoformat(),
|
||||
"created_at": self.created_at,
|
||||
"created_at_local": self.created_at_local,
|
||||
"has_database": self.has_database,
|
||||
"has_compose": self.has_compose,
|
||||
"database_name": self.database_name,
|
||||
"compose_files_count": self.compose_files_count,
|
||||
"warnings": self.warnings,
|
||||
"manifest": self.manifest,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackupRestoreResult:
|
||||
archive_name: str
|
||||
started_at: datetime
|
||||
completed_at: datetime
|
||||
database_restored: bool = False
|
||||
compose_files_restored: int = 0
|
||||
compose_target_dir: Optional[str] = None
|
||||
compose_pre_restore_archive: Optional[str] = None
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
def to_payload(self) -> dict[str, Any]:
|
||||
return {
|
||||
"archive_name": self.archive_name,
|
||||
"started_at": self.started_at.isoformat(),
|
||||
"completed_at": self.completed_at.isoformat(),
|
||||
"database_restored": self.database_restored,
|
||||
"compose_files_restored": self.compose_files_restored,
|
||||
"compose_target_dir": self.compose_target_dir,
|
||||
"compose_pre_restore_archive": self.compose_pre_restore_archive,
|
||||
"warnings": self.warnings,
|
||||
}
|
||||
|
||||
|
||||
class BackupRestoreService:
|
||||
def __init__(self, settings: Settings):
|
||||
self.settings = settings
|
||||
|
||||
def backup_dir(self) -> Path:
|
||||
path = Path(self.settings.BACKUP_DIR).expanduser()
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
def list_archives(self) -> list[BackupArchiveInfo]:
|
||||
backup_dir = self.backup_dir()
|
||||
archives = []
|
||||
for path in backup_dir.glob("*.zip"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
try:
|
||||
archives.append(self.inspect_archive(path))
|
||||
except BackupArchiveError as exc:
|
||||
logger.warning("Skipping invalid backup archive %s: %s", path, exc)
|
||||
return sorted(archives, key=lambda item: item.modified_at, reverse=True)
|
||||
|
||||
def archive_path_for_name(self, archive_name: str) -> Path:
|
||||
raw_name = str(archive_name or "").strip()
|
||||
safe_name = Path(raw_name).name
|
||||
if not raw_name or safe_name != raw_name or not SAFE_ARCHIVE_NAME_RE.fullmatch(safe_name):
|
||||
raise BackupArchiveError("Invalid archive name")
|
||||
|
||||
backup_dir = self.backup_dir().resolve()
|
||||
archive_path = (backup_dir / safe_name).resolve()
|
||||
try:
|
||||
archive_path.relative_to(backup_dir)
|
||||
except ValueError as exc:
|
||||
raise BackupArchiveError("Archive path escapes backup directory") from exc
|
||||
if not archive_path.is_file():
|
||||
raise BackupArchiveError("Archive does not exist")
|
||||
return archive_path
|
||||
|
||||
def inspect_archive(self, archive_path: Path) -> BackupArchiveInfo:
|
||||
if not zipfile.is_zipfile(archive_path):
|
||||
raise BackupArchiveError("Archive is not a valid ZIP file")
|
||||
|
||||
stat = archive_path.stat()
|
||||
warnings: list[str] = []
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
self._validate_zip_members(archive.infolist())
|
||||
manifest = self._read_manifest(archive)
|
||||
signature_valid = self._archive_signature_valid(manifest)
|
||||
signature_required = getattr(
|
||||
self.settings,
|
||||
"BACKUP_ARCHIVE_SIGNATURE_REQUIRED",
|
||||
True,
|
||||
)
|
||||
if signature_required and not signature_valid:
|
||||
raise BackupArchiveError("Archive manifest signature is not valid")
|
||||
if not signature_valid:
|
||||
warnings.append("manifest signature is not valid")
|
||||
has_database = self._find_database_dump_member(archive) is not None
|
||||
compose_members = self._compose_file_members(archive)
|
||||
|
||||
manifest_warnings = manifest.get("warnings")
|
||||
if isinstance(manifest_warnings, list):
|
||||
warnings.extend(str(item) for item in manifest_warnings if item)
|
||||
|
||||
postgres = manifest.get("postgres") if isinstance(manifest.get("postgres"), dict) else {}
|
||||
compose = manifest.get("compose") if isinstance(manifest.get("compose"), dict) else {}
|
||||
return BackupArchiveInfo(
|
||||
name=archive_path.name,
|
||||
path=archive_path,
|
||||
size_bytes=int(stat.st_size),
|
||||
modified_at=datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc),
|
||||
created_at=str(manifest.get("created_at") or "") or None,
|
||||
created_at_local=str(manifest.get("created_at_local") or "") or None,
|
||||
has_database=has_database,
|
||||
has_compose=bool(compose_members),
|
||||
database_name=str(postgres.get("database") or "") or None,
|
||||
compose_files_count=int(compose.get("files_count") or len(compose_members)),
|
||||
warnings=warnings,
|
||||
manifest=manifest,
|
||||
)
|
||||
|
||||
def import_uploaded_archive(
|
||||
self,
|
||||
temp_path: Path,
|
||||
original_filename: str = "",
|
||||
) -> BackupArchiveInfo:
|
||||
self._validate_archive_for_restore(temp_path)
|
||||
digest = self._file_digest(temp_path)
|
||||
stamp = datetime.now().astimezone().strftime("%Y%m%d-%H%M%S%z")
|
||||
safe_original = self._safe_original_stem(original_filename)
|
||||
archive_name = f"{BACKUP_FILENAME_PREFIX}uploaded-{stamp}-{digest}-{safe_original}.zip"
|
||||
target_path = self._unique_archive_path(archive_name)
|
||||
temp_path.replace(target_path)
|
||||
return self.inspect_archive(target_path)
|
||||
|
||||
async def restore_archive(
|
||||
self,
|
||||
archive_name: str,
|
||||
*,
|
||||
restore_database: bool,
|
||||
restore_compose: bool,
|
||||
) -> BackupRestoreResult:
|
||||
return await asyncio.to_thread(
|
||||
self.restore_archive_sync,
|
||||
archive_name,
|
||||
restore_database=restore_database,
|
||||
restore_compose=restore_compose,
|
||||
)
|
||||
|
||||
def restore_archive_sync(
|
||||
self,
|
||||
archive_name: str,
|
||||
*,
|
||||
restore_database: bool,
|
||||
restore_compose: bool,
|
||||
) -> BackupRestoreResult:
|
||||
if not restore_database and not restore_compose:
|
||||
raise BackupArchiveError("Select at least one restore target")
|
||||
|
||||
archive_path = self.archive_path_for_name(archive_name)
|
||||
self._validate_archive_for_restore(archive_path)
|
||||
started_at = datetime.now(timezone.utc)
|
||||
warnings: list[str] = []
|
||||
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix=f"restore-{archive_path.stem}-",
|
||||
dir=self.backup_dir(),
|
||||
) as tmp:
|
||||
temp_dir = Path(tmp)
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
self._validate_zip_members(archive.infolist())
|
||||
db_member = self._find_database_dump_member(archive) if restore_database else None
|
||||
compose_members = self._compose_file_members(archive) if restore_compose else []
|
||||
|
||||
if restore_database and db_member is None:
|
||||
raise BackupArchiveError("Archive does not contain a database dump")
|
||||
if restore_compose and not compose_members:
|
||||
raise BackupArchiveError("Archive does not contain compose files")
|
||||
|
||||
compose_target_dir: Optional[Path] = None
|
||||
compose_pre_restore_archive: Optional[Path] = None
|
||||
if restore_compose:
|
||||
compose_target_dir = self._compose_restore_target_dir()
|
||||
self._assert_compose_target_writable(compose_target_dir)
|
||||
compose_pre_restore_archive = self._snapshot_current_compose(compose_target_dir)
|
||||
|
||||
database_restored = False
|
||||
if db_member is not None:
|
||||
dump_path = self._extract_database_dump(archive, db_member, temp_dir)
|
||||
self._run_pg_restore(dump_path)
|
||||
database_restored = True
|
||||
|
||||
compose_files_restored = 0
|
||||
if compose_target_dir is not None:
|
||||
compose_files_restored = self._restore_compose_members(
|
||||
archive,
|
||||
compose_members,
|
||||
compose_target_dir,
|
||||
)
|
||||
|
||||
return BackupRestoreResult(
|
||||
archive_name=archive_path.name,
|
||||
started_at=started_at,
|
||||
completed_at=datetime.now(timezone.utc),
|
||||
database_restored=database_restored,
|
||||
compose_files_restored=compose_files_restored,
|
||||
compose_target_dir=str(compose_target_dir) if compose_target_dir else None,
|
||||
compose_pre_restore_archive=str(compose_pre_restore_archive)
|
||||
if compose_pre_restore_archive
|
||||
else None,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
def _run_pg_restore(self, dump_path: Path) -> None:
|
||||
pg_restore_path = str(getattr(self.settings, "BACKUP_PG_RESTORE_PATH", "pg_restore") or "")
|
||||
pg_restore_path = pg_restore_path or "pg_restore"
|
||||
if shutil.which(pg_restore_path) is None and Path(pg_restore_path).name == pg_restore_path:
|
||||
raise BackupRestoreError(
|
||||
"pg_restore executable was not found. Rebuild the backend image with "
|
||||
"PostgreSQL client tools."
|
||||
)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["PGPASSWORD"] = self.settings.POSTGRES_PASSWORD
|
||||
command = [
|
||||
pg_restore_path,
|
||||
"-h",
|
||||
self.settings.POSTGRES_HOST,
|
||||
"-p",
|
||||
str(self.settings.POSTGRES_PORT),
|
||||
"-U",
|
||||
self.settings.POSTGRES_USER,
|
||||
"-d",
|
||||
self.settings.POSTGRES_DB,
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
"--no-owner",
|
||||
"--no-privileges",
|
||||
str(dump_path),
|
||||
]
|
||||
timeout = max(
|
||||
30,
|
||||
int(
|
||||
getattr(
|
||||
self.settings,
|
||||
"BACKUP_PG_RESTORE_TIMEOUT_SECONDS",
|
||||
self.settings.BACKUP_PG_DUMP_TIMEOUT_SECONDS,
|
||||
)
|
||||
or 1800
|
||||
),
|
||||
)
|
||||
result = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
stderr = (result.stderr or result.stdout or "").strip()
|
||||
raise BackupRestoreError(
|
||||
f"pg_restore failed with exit code {result.returncode}: {stderr[:500]}"
|
||||
)
|
||||
|
||||
def _compose_restore_target_dir(self) -> Path:
|
||||
target_raw = (
|
||||
getattr(self.settings, "BACKUP_COMPOSE_RESTORE_DIR", None)
|
||||
or self.settings.BACKUP_COMPOSE_SOURCE_DIR
|
||||
or ""
|
||||
)
|
||||
if not str(target_raw).strip():
|
||||
raise BackupArchiveError("Compose restore directory is not configured")
|
||||
return Path(str(target_raw)).expanduser()
|
||||
|
||||
def _assert_compose_target_writable(self, target_dir: Path) -> None:
|
||||
if not target_dir.exists() or not target_dir.is_dir():
|
||||
raise BackupArchiveError(
|
||||
f"Compose restore directory is unavailable: {target_dir}. "
|
||||
"Mount the compose folder into the backend container."
|
||||
)
|
||||
probe = target_dir / f".restore-write-test-{os.getpid()}"
|
||||
try:
|
||||
probe.write_text("", encoding="utf-8")
|
||||
probe.unlink()
|
||||
except OSError as exc:
|
||||
raise BackupArchiveError(
|
||||
f"Compose restore directory is not writable: {target_dir}"
|
||||
) from exc
|
||||
|
||||
def _snapshot_current_compose(self, target_dir: Path) -> Optional[Path]:
|
||||
stamp = datetime.now().astimezone().strftime("%Y%m%d-%H%M%S%z")
|
||||
archive_path = self.backup_dir() / f"{COMPOSE_PRE_RESTORE_PREFIX}{stamp}.zip"
|
||||
excluded_dirs = self._compose_excluded_dirs()
|
||||
files_count = 0
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix=f"{COMPOSE_PRE_RESTORE_PREFIX}{stamp}-",
|
||||
dir=self.backup_dir(),
|
||||
) as tmp:
|
||||
staging_dir = Path(tmp)
|
||||
compose_dir = staging_dir / "compose"
|
||||
for path in sorted(target_dir.rglob("*")):
|
||||
relative = path.relative_to(target_dir)
|
||||
if any(part in excluded_dirs for part in relative.parts):
|
||||
continue
|
||||
if path.is_dir() or path.is_symlink():
|
||||
continue
|
||||
destination = compose_dir / relative
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(path, destination)
|
||||
files_count += 1
|
||||
if files_count <= 0:
|
||||
return None
|
||||
|
||||
completed_at = datetime.now(timezone.utc)
|
||||
manifest = {
|
||||
"app": BACKUP_APP_ID,
|
||||
"format_version": BACKUP_FORMAT_VERSION,
|
||||
"type": "compose-pre-restore",
|
||||
"created_at": completed_at.isoformat(),
|
||||
"created_at_local": completed_at.astimezone().isoformat(),
|
||||
"postgres": {
|
||||
"database": self.settings.POSTGRES_DB,
|
||||
"included": False,
|
||||
},
|
||||
"compose": {
|
||||
"source_dir": str(target_dir),
|
||||
"included": True,
|
||||
"files_count": files_count,
|
||||
},
|
||||
"warnings": [],
|
||||
}
|
||||
attach_archive_integrity(
|
||||
manifest,
|
||||
file_records=build_file_records(staging_dir),
|
||||
settings=self.settings,
|
||||
)
|
||||
write_manifest(staging_dir, manifest)
|
||||
tmp_archive = archive_path.with_name(f"{archive_path.name}.tmp")
|
||||
try:
|
||||
write_zip_from_directory(staging_dir, tmp_archive)
|
||||
tmp_archive.replace(archive_path)
|
||||
finally:
|
||||
if tmp_archive.exists():
|
||||
try:
|
||||
tmp_archive.unlink()
|
||||
except OSError:
|
||||
logger.warning("Failed to remove temporary snapshot %s", tmp_archive)
|
||||
return archive_path
|
||||
|
||||
def _restore_compose_members(
|
||||
self,
|
||||
archive: zipfile.ZipFile,
|
||||
members: list[zipfile.ZipInfo],
|
||||
target_dir: Path,
|
||||
) -> int:
|
||||
target_root = target_dir.resolve()
|
||||
restored = 0
|
||||
for member in members:
|
||||
relative = PurePosixPath(member.filename).relative_to("compose")
|
||||
destination = target_root.joinpath(*relative.parts).resolve()
|
||||
try:
|
||||
destination.relative_to(target_root)
|
||||
except ValueError as exc:
|
||||
raise BackupArchiveError(
|
||||
f"Unsafe compose archive member: {member.filename}"
|
||||
) from exc
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp_destination = destination.with_name(
|
||||
f".{destination.name}.restore-{os.getpid()}.tmp"
|
||||
)
|
||||
try:
|
||||
with archive.open(member) as source, temp_destination.open("wb") as target:
|
||||
shutil.copyfileobj(source, target)
|
||||
temp_destination.replace(destination)
|
||||
finally:
|
||||
if temp_destination.exists():
|
||||
try:
|
||||
temp_destination.unlink()
|
||||
except OSError:
|
||||
logger.warning(
|
||||
"Failed to remove temporary restore file %s",
|
||||
temp_destination,
|
||||
)
|
||||
restored += 1
|
||||
return restored
|
||||
|
||||
def _extract_database_dump(
|
||||
self,
|
||||
archive: zipfile.ZipFile,
|
||||
member: zipfile.ZipInfo,
|
||||
temp_dir: Path,
|
||||
) -> Path:
|
||||
dump_dir = temp_dir / "database"
|
||||
dump_dir.mkdir(parents=True, exist_ok=True)
|
||||
dump_path = dump_dir / Path(member.filename).name
|
||||
with archive.open(member) as source, dump_path.open("wb") as target:
|
||||
shutil.copyfileobj(source, target)
|
||||
return dump_path
|
||||
|
||||
def _find_database_dump_member(self, archive: zipfile.ZipFile) -> Optional[zipfile.ZipInfo]:
|
||||
candidates = [
|
||||
item
|
||||
for item in archive.infolist()
|
||||
if not item.is_dir()
|
||||
and item.filename.startswith("database/")
|
||||
and PurePosixPath(item.filename).suffix.lower() in {".dump", ".backup"}
|
||||
]
|
||||
return sorted(candidates, key=lambda item: item.filename)[0] if candidates else None
|
||||
|
||||
def _compose_file_members(self, archive: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
|
||||
members = [
|
||||
item
|
||||
for item in archive.infolist()
|
||||
if not item.is_dir() and item.filename.startswith("compose/")
|
||||
]
|
||||
self._validate_compose_members(members)
|
||||
return members
|
||||
|
||||
def _validate_zip_members(self, members: list[zipfile.ZipInfo]) -> None:
|
||||
if len(members) > BACKUP_MAX_MEMBERS:
|
||||
raise BackupArchiveError("Archive contains too many files")
|
||||
|
||||
seen: set[str] = set()
|
||||
total_size = 0
|
||||
for member in members:
|
||||
filename = member.filename
|
||||
if "\\" in filename or "\x00" in filename:
|
||||
raise BackupArchiveError(f"Unsafe archive member path: {filename}")
|
||||
path = PurePosixPath(member.filename)
|
||||
if (
|
||||
not path.parts
|
||||
or path.is_absolute()
|
||||
or ".." in path.parts
|
||||
or any(part in {"", "."} for part in path.parts)
|
||||
):
|
||||
raise BackupArchiveError(f"Unsafe archive member path: {member.filename}")
|
||||
if member.is_dir():
|
||||
continue
|
||||
if filename in seen:
|
||||
raise BackupArchiveError(f"Duplicate archive member path: {filename}")
|
||||
seen.add(filename)
|
||||
if member.file_size > BACKUP_MAX_MEMBER_BYTES:
|
||||
raise BackupArchiveError(f"Archive member is too large: {filename}")
|
||||
total_size += int(member.file_size)
|
||||
if total_size > BACKUP_MAX_UNCOMPRESSED_BYTES:
|
||||
raise BackupArchiveError("Archive uncompressed size is too large")
|
||||
compressed = max(1, int(member.compress_size or 1))
|
||||
ratio = int(member.file_size) / compressed
|
||||
if (
|
||||
member.file_size >= BACKUP_ZIP_BOMB_MIN_BYTES
|
||||
and ratio > BACKUP_MAX_COMPRESSION_RATIO
|
||||
):
|
||||
raise BackupArchiveError(
|
||||
f"Archive member compression ratio is too high: {filename}"
|
||||
)
|
||||
|
||||
def _validate_compose_members(self, members: list[zipfile.ZipInfo]) -> None:
|
||||
total_size = 0
|
||||
for member in members:
|
||||
if member.file_size > BACKUP_MAX_COMPOSE_MEMBER_BYTES:
|
||||
raise BackupArchiveError(f"Compose archive member is too large: {member.filename}")
|
||||
total_size += int(member.file_size)
|
||||
if total_size > BACKUP_MAX_COMPOSE_BYTES:
|
||||
raise BackupArchiveError("Compose archive contents are too large")
|
||||
|
||||
def _read_manifest(self, archive: zipfile.ZipFile) -> dict[str, Any]:
|
||||
if BACKUP_MANIFEST_NAME not in archive.namelist():
|
||||
raise BackupArchiveError("Archive does not contain manifest.json")
|
||||
try:
|
||||
manifest = json.loads(archive.read(BACKUP_MANIFEST_NAME).decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise BackupArchiveError("manifest.json is not valid JSON") from exc
|
||||
if not isinstance(manifest, dict):
|
||||
raise BackupArchiveError("manifest.json must contain an object")
|
||||
if manifest.get("app") != BACKUP_APP_ID:
|
||||
raise BackupArchiveError("Archive manifest belongs to another application")
|
||||
try:
|
||||
format_version = int(manifest.get("format_version") or 0)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise BackupArchiveError("Archive manifest format is not supported") from exc
|
||||
if format_version != BACKUP_FORMAT_VERSION:
|
||||
raise BackupArchiveError("Archive manifest format is not supported")
|
||||
return manifest
|
||||
|
||||
def _archive_signature_valid(self, manifest: dict[str, Any]) -> bool:
|
||||
return verify_manifest_signature(manifest, self.settings)
|
||||
|
||||
def _validate_archive_for_restore(self, archive_path: Path) -> None:
|
||||
if not zipfile.is_zipfile(archive_path):
|
||||
raise BackupArchiveError("Archive is not a valid ZIP file")
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
self._validate_zip_members(archive.infolist())
|
||||
manifest = self._read_manifest(archive)
|
||||
if getattr(self.settings, "BACKUP_ARCHIVE_SIGNATURE_REQUIRED", True):
|
||||
if not self._archive_signature_valid(manifest):
|
||||
raise BackupArchiveError("Archive manifest signature is not valid")
|
||||
self._validate_archive_integrity(archive, manifest)
|
||||
|
||||
def _validate_archive_integrity(
|
||||
self,
|
||||
archive: zipfile.ZipFile,
|
||||
manifest: dict[str, Any],
|
||||
) -> None:
|
||||
archive_manifest = (
|
||||
manifest.get("archive") if isinstance(manifest.get("archive"), dict) else {}
|
||||
)
|
||||
file_records = archive_manifest.get("files")
|
||||
if not isinstance(file_records, list):
|
||||
raise BackupArchiveError("Archive manifest does not contain file checksums")
|
||||
|
||||
expected: dict[str, dict[str, Any]] = {}
|
||||
for record in file_records:
|
||||
if not isinstance(record, dict):
|
||||
raise BackupArchiveError("Archive manifest contains invalid file record")
|
||||
filename = str(record.get("path") or "")
|
||||
if not filename:
|
||||
raise BackupArchiveError("Archive manifest contains empty file path")
|
||||
if filename in expected:
|
||||
raise BackupArchiveError(
|
||||
f"Archive manifest contains duplicate file path: {filename}"
|
||||
)
|
||||
expected[filename] = record
|
||||
|
||||
actual = {
|
||||
item.filename
|
||||
for item in archive.infolist()
|
||||
if not item.is_dir() and item.filename != BACKUP_MANIFEST_NAME
|
||||
}
|
||||
if actual != set(expected):
|
||||
raise BackupArchiveError("Archive contents do not match manifest")
|
||||
|
||||
for info in archive.infolist():
|
||||
if info.is_dir() or info.filename == BACKUP_MANIFEST_NAME:
|
||||
continue
|
||||
record = expected[info.filename]
|
||||
try:
|
||||
expected_size = int(record.get("size_bytes") or -1)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise BackupArchiveError(
|
||||
f"Archive manifest size is invalid: {info.filename}"
|
||||
) from exc
|
||||
expected_hash = str(record.get("sha256") or "")
|
||||
if expected_size != int(info.file_size):
|
||||
raise BackupArchiveError(
|
||||
f"Archive member size does not match manifest: {info.filename}"
|
||||
)
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", expected_hash):
|
||||
raise BackupArchiveError(f"Archive manifest checksum is invalid: {info.filename}")
|
||||
digest = hashlib.sha256()
|
||||
with archive.open(info) as source:
|
||||
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
if not hmac.compare_digest(digest.hexdigest(), expected_hash):
|
||||
raise BackupArchiveError(
|
||||
f"Archive member checksum does not match manifest: {info.filename}"
|
||||
)
|
||||
|
||||
def _compose_excluded_dirs(self) -> set[str]:
|
||||
configured = self._split_csv(self.settings.BACKUP_COMPOSE_EXCLUDE_DIRS)
|
||||
return DEFAULT_COMPOSE_EXCLUDED_DIRS | set(configured)
|
||||
|
||||
@staticmethod
|
||||
def _split_csv(value: Optional[str]) -> list[str]:
|
||||
if not value:
|
||||
return []
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
|
||||
@staticmethod
|
||||
def _file_digest(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()[:16]
|
||||
|
||||
@staticmethod
|
||||
def _safe_original_stem(filename: str) -> str:
|
||||
stem = Path(str(filename or "backup")).stem
|
||||
safe = re.sub(r"[^A-Za-z0-9_.+-]+", "-", stem).strip(".-")
|
||||
return (safe or "backup")[:72]
|
||||
|
||||
def _unique_archive_path(self, archive_name: str) -> Path:
|
||||
backup_dir = self.backup_dir()
|
||||
stem = Path(archive_name).stem
|
||||
suffix = Path(archive_name).suffix
|
||||
candidate = backup_dir / archive_name
|
||||
counter = 2
|
||||
while candidate.exists():
|
||||
candidate = backup_dir / f"{stem}-{counter}{suffix}"
|
||||
counter += 1
|
||||
return candidate
|
||||
@@ -0,0 +1,419 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Optional
|
||||
|
||||
from aiogram import Bot
|
||||
from aiogram.types import FSInputFile
|
||||
|
||||
from bot.infra.redis import redis_lock
|
||||
from bot.services.backup_archive import (
|
||||
BACKUP_APP_ID,
|
||||
BACKUP_FILENAME_PREFIX,
|
||||
BACKUP_FORMAT_VERSION,
|
||||
attach_archive_integrity,
|
||||
build_file_records,
|
||||
write_manifest,
|
||||
write_zip_from_directory,
|
||||
)
|
||||
from config.settings import Settings
|
||||
|
||||
COMPOSE_MARKER_FILES = {
|
||||
"compose.yaml",
|
||||
"compose.yml",
|
||||
"docker-compose.yaml",
|
||||
"docker-compose.yml",
|
||||
}
|
||||
DEFAULT_COMPOSE_EXCLUDED_DIRS = {
|
||||
".git",
|
||||
".pytest_cache",
|
||||
".ruff_cache",
|
||||
"__pycache__",
|
||||
"backups",
|
||||
"node_modules",
|
||||
"postgres-data",
|
||||
"redis-data",
|
||||
"shop-data",
|
||||
}
|
||||
BACKUP_RUNTIME_SETTING_KEYS = {
|
||||
"BACKUP_ENABLED",
|
||||
"BACKUP_CHAT_ID",
|
||||
"BACKUP_THREAD_ID",
|
||||
"BACKUP_INTERVAL_SECONDS",
|
||||
"BACKUP_LOCAL_RETENTION",
|
||||
"BACKUP_POSTGRES_DUMP_ENABLED",
|
||||
"BACKUP_PG_DUMP_PATH",
|
||||
"BACKUP_PG_DUMP_TIMEOUT_SECONDS",
|
||||
"BACKUP_COMPOSE_ENABLED",
|
||||
"BACKUP_COMPOSE_SOURCE_DIR",
|
||||
"BACKUP_COMPOSE_EXCLUDE_DIRS",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackupResult:
|
||||
archive_path: Path
|
||||
started_at: datetime
|
||||
completed_at: datetime
|
||||
db_dump_included: bool
|
||||
compose_files_count: int
|
||||
size_bytes: int
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class BackupWorker:
|
||||
SETTINGS_REFRESH_SECONDS = 60
|
||||
|
||||
def __init__(self, settings: Settings, bot: Bot, session_factory=None):
|
||||
self.settings = settings
|
||||
self.bot = bot
|
||||
self.session_factory = session_factory
|
||||
|
||||
async def run(self) -> None:
|
||||
while True:
|
||||
await self._refresh_settings()
|
||||
if not self.settings.BACKUP_ENABLED:
|
||||
await asyncio.sleep(self.SETTINGS_REFRESH_SECONDS)
|
||||
continue
|
||||
|
||||
interval = self._interval_seconds()
|
||||
delay_seconds = self._seconds_until_next_slot(interval)
|
||||
if delay_seconds > 0:
|
||||
should_run = await self._sleep_until_next_slot(delay_seconds, interval)
|
||||
if not should_run:
|
||||
continue
|
||||
|
||||
await self._refresh_settings()
|
||||
if not self.settings.BACKUP_ENABLED:
|
||||
continue
|
||||
|
||||
try:
|
||||
ttl_seconds = max(
|
||||
60,
|
||||
int(getattr(self.settings, "BACKUP_LOCK_TTL_SECONDS", 7200) or 7200),
|
||||
)
|
||||
async with redis_lock(
|
||||
self.settings,
|
||||
"backup-worker",
|
||||
ttl_seconds=ttl_seconds,
|
||||
) as acquired:
|
||||
if acquired:
|
||||
started = time.monotonic()
|
||||
result = await self.create_and_send_backup()
|
||||
logging.info(
|
||||
"metric worker_tick_duration_seconds=%.3f worker=backup size_bytes=%s",
|
||||
time.monotonic() - started,
|
||||
result.size_bytes,
|
||||
)
|
||||
else:
|
||||
logging.info(
|
||||
"Backup worker tick skipped because another worker holds the lock"
|
||||
)
|
||||
except Exception as exc:
|
||||
logging.exception("Backup worker tick failed")
|
||||
await self._notify_failure(exc)
|
||||
|
||||
async def create_and_send_backup(self) -> BackupResult:
|
||||
result = await self.create_backup()
|
||||
try:
|
||||
await self.send_backup(result)
|
||||
finally:
|
||||
self.prune_old_backups()
|
||||
return result
|
||||
|
||||
async def create_backup(self) -> BackupResult:
|
||||
started_at = datetime.now(timezone.utc)
|
||||
stamp = datetime.now().astimezone().strftime("%Y%m%d-%H%M%S%z")
|
||||
archive_name = f"{BACKUP_FILENAME_PREFIX}{stamp}.zip"
|
||||
backup_dir = Path(self.settings.BACKUP_DIR).expanduser()
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix=f"{BACKUP_FILENAME_PREFIX}{stamp}-",
|
||||
dir=backup_dir,
|
||||
) as tmp:
|
||||
staging_dir = Path(tmp)
|
||||
warnings: list[str] = []
|
||||
db_dump_included = False
|
||||
compose_files_count = 0
|
||||
|
||||
if self.settings.BACKUP_POSTGRES_DUMP_ENABLED:
|
||||
dump_dir = staging_dir / "database"
|
||||
dump_dir.mkdir(parents=True, exist_ok=True)
|
||||
dump_path = dump_dir / f"{self.settings.POSTGRES_DB}.dump"
|
||||
await self._dump_database(dump_path)
|
||||
db_dump_included = True
|
||||
|
||||
if self.settings.BACKUP_COMPOSE_ENABLED:
|
||||
compose_files_count = self._stage_compose_source(staging_dir / "compose", warnings)
|
||||
|
||||
completed_at = datetime.now(timezone.utc)
|
||||
manifest = {
|
||||
"app": BACKUP_APP_ID,
|
||||
"format_version": BACKUP_FORMAT_VERSION,
|
||||
"type": "scheduled",
|
||||
"created_at": completed_at.isoformat(),
|
||||
"created_at_local": completed_at.astimezone().isoformat(),
|
||||
"postgres": {
|
||||
"host": self.settings.POSTGRES_HOST,
|
||||
"port": self.settings.POSTGRES_PORT,
|
||||
"database": self.settings.POSTGRES_DB,
|
||||
"user": self.settings.POSTGRES_USER,
|
||||
"dump_format": "pg_dump custom",
|
||||
"included": db_dump_included,
|
||||
},
|
||||
"compose": {
|
||||
"source_dir": self.settings.BACKUP_COMPOSE_SOURCE_DIR,
|
||||
"included": compose_files_count > 0,
|
||||
"files_count": compose_files_count,
|
||||
},
|
||||
"warnings": warnings,
|
||||
}
|
||||
attach_archive_integrity(
|
||||
manifest,
|
||||
file_records=build_file_records(staging_dir),
|
||||
settings=self.settings,
|
||||
)
|
||||
write_manifest(staging_dir, manifest)
|
||||
|
||||
tmp_archive = backup_dir / f"{archive_name}.tmp"
|
||||
archive_path = backup_dir / archive_name
|
||||
write_zip_from_directory(staging_dir, tmp_archive)
|
||||
tmp_archive.replace(archive_path)
|
||||
|
||||
return BackupResult(
|
||||
archive_path=archive_path,
|
||||
started_at=started_at,
|
||||
completed_at=completed_at,
|
||||
db_dump_included=db_dump_included,
|
||||
compose_files_count=compose_files_count,
|
||||
size_bytes=archive_path.stat().st_size,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
async def _dump_database(self, dump_path: Path) -> None:
|
||||
await asyncio.to_thread(self._run_pg_dump, dump_path)
|
||||
|
||||
def _run_pg_dump(self, dump_path: Path) -> None:
|
||||
pg_dump_path = str(self.settings.BACKUP_PG_DUMP_PATH or "pg_dump")
|
||||
if shutil.which(pg_dump_path) is None and Path(pg_dump_path).name == pg_dump_path:
|
||||
raise RuntimeError(
|
||||
"pg_dump executable was not found. Rebuild the worker image with "
|
||||
"PostgreSQL client tools."
|
||||
)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["PGPASSWORD"] = self.settings.POSTGRES_PASSWORD
|
||||
command = [
|
||||
pg_dump_path,
|
||||
"-h",
|
||||
self.settings.POSTGRES_HOST,
|
||||
"-p",
|
||||
str(self.settings.POSTGRES_PORT),
|
||||
"-U",
|
||||
self.settings.POSTGRES_USER,
|
||||
"-d",
|
||||
self.settings.POSTGRES_DB,
|
||||
"--format=custom",
|
||||
"--no-owner",
|
||||
"--no-privileges",
|
||||
"--file",
|
||||
str(dump_path),
|
||||
]
|
||||
timeout = max(30, int(self.settings.BACKUP_PG_DUMP_TIMEOUT_SECONDS or 1800))
|
||||
result = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
stderr = (result.stderr or result.stdout or "").strip()
|
||||
raise RuntimeError(f"pg_dump failed with exit code {result.returncode}: {stderr[:500]}")
|
||||
|
||||
def _stage_compose_source(self, target_dir: Path, warnings: list[str]) -> int:
|
||||
source_raw = (self.settings.BACKUP_COMPOSE_SOURCE_DIR or "").strip()
|
||||
if not source_raw:
|
||||
warnings.append("Compose source directory is not configured.")
|
||||
return 0
|
||||
|
||||
source_dir = Path(source_raw).expanduser()
|
||||
if not source_dir.exists() or not source_dir.is_dir():
|
||||
warnings.append(f"Compose source directory is unavailable: {source_dir}")
|
||||
return 0
|
||||
|
||||
if not any((source_dir / marker).is_file() for marker in COMPOSE_MARKER_FILES):
|
||||
warnings.append(f"Compose source directory has no compose file marker: {source_dir}")
|
||||
|
||||
excluded_dirs = self._compose_excluded_dirs()
|
||||
files_count = 0
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for path in source_dir.rglob("*"):
|
||||
relative = path.relative_to(source_dir)
|
||||
if any(part in excluded_dirs for part in relative.parts):
|
||||
continue
|
||||
if path.is_dir() or path.is_symlink():
|
||||
continue
|
||||
if path.name.startswith(f"{BACKUP_FILENAME_PREFIX}") and path.suffix == ".zip":
|
||||
continue
|
||||
destination = target_dir / relative
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
shutil.copy2(path, destination)
|
||||
files_count += 1
|
||||
except OSError as exc:
|
||||
warnings.append(f"Skipped compose file {relative.as_posix()}: {exc}")
|
||||
|
||||
return files_count
|
||||
|
||||
def _compose_excluded_dirs(self) -> set[str]:
|
||||
configured = self._split_csv(self.settings.BACKUP_COMPOSE_EXCLUDE_DIRS)
|
||||
return DEFAULT_COMPOSE_EXCLUDED_DIRS | set(configured)
|
||||
|
||||
@staticmethod
|
||||
def _split_csv(value: Optional[str]) -> list[str]:
|
||||
if not value:
|
||||
return []
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
|
||||
async def send_backup(self, result: BackupResult) -> None:
|
||||
chat_id = self._target_chat_id()
|
||||
if chat_id is None:
|
||||
logging.warning(
|
||||
"Backup archive created at %s but BACKUP_CHAT_ID/LOG_CHAT_ID is not configured",
|
||||
result.archive_path,
|
||||
)
|
||||
return
|
||||
|
||||
kwargs = {
|
||||
"chat_id": chat_id,
|
||||
"document": FSInputFile(result.archive_path),
|
||||
"caption": self._caption(result),
|
||||
}
|
||||
thread_id = self._target_thread_id()
|
||||
if thread_id is not None:
|
||||
kwargs["message_thread_id"] = thread_id
|
||||
await self.bot.send_document(**kwargs)
|
||||
|
||||
def prune_old_backups(self) -> None:
|
||||
retention = int(getattr(self.settings, "BACKUP_LOCAL_RETENTION", 3) or 0)
|
||||
if retention <= 0:
|
||||
return
|
||||
|
||||
backup_dir = Path(self.settings.BACKUP_DIR).expanduser()
|
||||
archives = sorted(
|
||||
backup_dir.glob(f"{BACKUP_FILENAME_PREFIX}*.zip"),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
for archive in archives[retention:]:
|
||||
try:
|
||||
archive.unlink()
|
||||
except OSError:
|
||||
logging.exception("Failed to delete old backup archive %s", archive)
|
||||
|
||||
def _target_chat_id(self) -> Optional[int]:
|
||||
return self.settings.BACKUP_CHAT_ID or self.settings.LOG_CHAT_ID
|
||||
|
||||
def _target_thread_id(self) -> Optional[int]:
|
||||
return self.settings.BACKUP_THREAD_ID or self.settings.LOG_THREAD_ID
|
||||
|
||||
def _caption(self, result: BackupResult) -> str:
|
||||
completed_at = result.completed_at.astimezone()
|
||||
lines = [
|
||||
"Remnawave Minishop backup",
|
||||
f"Created: {completed_at.strftime('%Y-%m-%d %H:%M:%S %Z')}",
|
||||
f"Database dump: {'yes' if result.db_dump_included else 'no'}",
|
||||
f"Compose files: {result.compose_files_count}",
|
||||
f"Archive size: {self._human_size(result.size_bytes)}",
|
||||
]
|
||||
if result.warnings:
|
||||
lines.append(f"Warnings: {len(result.warnings)}")
|
||||
return "\n".join(lines)
|
||||
|
||||
@staticmethod
|
||||
def _human_size(size_bytes: int) -> str:
|
||||
units: Iterable[str] = ("B", "KB", "MB", "GB")
|
||||
size = float(size_bytes)
|
||||
unit = "B"
|
||||
for unit in units:
|
||||
if size < 1024 or unit == "GB":
|
||||
break
|
||||
size /= 1024
|
||||
if unit == "B":
|
||||
return f"{int(size)} {unit}"
|
||||
return f"{size:.1f} {unit}"
|
||||
|
||||
async def _refresh_settings(self) -> None:
|
||||
if self.session_factory is None:
|
||||
return
|
||||
try:
|
||||
from bot.services.settings_override_service import refresh_overrides_from_db
|
||||
|
||||
await refresh_overrides_from_db(
|
||||
self.settings,
|
||||
self.session_factory,
|
||||
keys=BACKUP_RUNTIME_SETTING_KEYS,
|
||||
)
|
||||
except Exception:
|
||||
logging.exception("Failed to refresh backup settings from DB")
|
||||
|
||||
def _interval_seconds(self) -> int:
|
||||
try:
|
||||
interval = int(self.settings.BACKUP_INTERVAL_SECONDS or 0)
|
||||
except (TypeError, ValueError):
|
||||
interval = 0
|
||||
return max(60, interval)
|
||||
|
||||
def _seconds_until_next_slot(self, interval_seconds: int) -> float:
|
||||
now = datetime.now().astimezone()
|
||||
if interval_seconds <= 0:
|
||||
return 0.0
|
||||
if interval_seconds <= 24 * 60 * 60:
|
||||
midnight = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
elapsed = (now - midnight).total_seconds()
|
||||
remainder = elapsed % interval_seconds
|
||||
else:
|
||||
remainder = time.time() % interval_seconds
|
||||
if remainder < 0.5:
|
||||
return 0.0
|
||||
return max(0.0, interval_seconds - remainder)
|
||||
|
||||
async def _sleep_until_next_slot(self, delay_seconds: float, interval_seconds: int) -> bool:
|
||||
deadline = datetime.now(timezone.utc) + timedelta(seconds=delay_seconds)
|
||||
while True:
|
||||
remaining = (deadline - datetime.now(timezone.utc)).total_seconds()
|
||||
if remaining <= 0:
|
||||
return True
|
||||
await asyncio.sleep(min(remaining, self.SETTINGS_REFRESH_SECONDS))
|
||||
await self._refresh_settings()
|
||||
if not self.settings.BACKUP_ENABLED:
|
||||
return False
|
||||
if self._interval_seconds() != interval_seconds:
|
||||
return False
|
||||
|
||||
async def _notify_failure(self, exc: Exception) -> None:
|
||||
chat_id = self._target_chat_id()
|
||||
if chat_id is None:
|
||||
return
|
||||
kwargs = {
|
||||
"chat_id": chat_id,
|
||||
"text": f"Remnawave Minishop backup failed: {type(exc).__name__}. Check worker logs.",
|
||||
}
|
||||
thread_id = self._target_thread_id()
|
||||
if thread_id is not None:
|
||||
kwargs["message_thread_id"] = thread_id
|
||||
try:
|
||||
await self.bot.send_message(**kwargs)
|
||||
except Exception:
|
||||
logging.exception("Failed to send backup failure notification")
|
||||
@@ -53,6 +53,7 @@ LOCALE_GROUPS = [
|
||||
"admin_sort_",
|
||||
"admin_status_",
|
||||
"admin_badge_",
|
||||
"admin_backups_",
|
||||
"admin_aria_",
|
||||
"admin_search",
|
||||
"admin_clear",
|
||||
@@ -257,6 +258,7 @@ LOCALE_GROUPS = [
|
||||
"audience": "internal",
|
||||
"prefixes": (
|
||||
"admin_settings_field_log_",
|
||||
"admin_settings_field_backup_",
|
||||
"admin_settings_field_support_",
|
||||
"admin_settings_field_subscription_notifications_",
|
||||
"admin_settings_field_subscription_notify_",
|
||||
|
||||
@@ -263,6 +263,35 @@ async def load_overrides_from_db(settings: Settings, async_session_factory: sess
|
||||
return applied
|
||||
|
||||
|
||||
async def refresh_overrides_from_db(
|
||||
settings: Settings,
|
||||
async_session_factory: sessionmaker,
|
||||
*,
|
||||
keys: Optional[set[str]] = None,
|
||||
) -> int:
|
||||
"""Refresh already-known runtime overrides without startup restore side effects."""
|
||||
|
||||
try:
|
||||
async with async_session_factory() as session:
|
||||
overrides = await app_settings_dal.get_all_overrides(session)
|
||||
except Exception as exc:
|
||||
logger.warning("Could not refresh setting overrides from DB: %s", exc)
|
||||
return 0
|
||||
if keys is not None:
|
||||
try:
|
||||
env_only = Settings()
|
||||
for key in keys:
|
||||
if key in overrides:
|
||||
continue
|
||||
attr_name = _resolve_attribute_name(env_only, key)
|
||||
if attr_name and hasattr(env_only, attr_name):
|
||||
setattr(settings, attr_name, getattr(env_only, attr_name))
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to restore env defaults while refreshing overrides: %s", exc)
|
||||
overrides = {key: value for key, value in overrides.items() if key in keys}
|
||||
return apply_overrides(settings, overrides)
|
||||
|
||||
|
||||
async def update_overrides(
|
||||
settings: Settings,
|
||||
async_session_factory: sessionmaker,
|
||||
|
||||
Reference in New Issue
Block a user