From 5e47a1496bbd4d0e42d8bd90f2a2da377971279f Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 00:13:54 +0300 Subject: [PATCH 01/11] feat: add Remnashop migration import Add the Remnashop legacy importer, compatibility tables, admin toggles, referral and promo lookup compatibility, and tests for the migration flow. --- .../bot/app/web/admin_settings_manifest.py | 33 + backend/bot/app/web/webapp/auth.py | 92 +- backend/bot/handlers/user/start.py | 89 +- backend/bot/services/promo_code_service.py | 18 +- backend/config/settings.py | 18 + backend/db/dal/promo_code_dal.py | 48 +- backend/db/dal/user_dal.py | 99 +- backend/db/migrator.py | 78 ++ backend/db/models.py | 33 +- backend/scripts/__init__.py | 2 + backend/scripts/import_legacy.py | 1164 +++++++++++++++++ .../src/admin/sections/SettingsSection.svelte | 1 + locales/en.json | 10 + locales/ru.json | 10 + tests/test_admin_settings_manifest_i18n.py | 28 + tests/test_remnashop_import.py | 44 + tests/test_support_migration.py | 23 +- tests/test_user_dal.py | 4 + 18 files changed, 1731 insertions(+), 63 deletions(-) create mode 100644 backend/scripts/__init__.py create mode 100644 backend/scripts/import_legacy.py create mode 100644 tests/test_remnashop_import.py diff --git a/backend/bot/app/web/admin_settings_manifest.py b/backend/bot/app/web/admin_settings_manifest.py index 9c86789..cd8680e 100644 --- a/backend/bot/app/web/admin_settings_manifest.py +++ b/backend/bot/app/web/admin_settings_manifest.py @@ -402,6 +402,38 @@ SETTINGS_MANIFEST: List[SettingField] = [ "REFERRAL_WELCOME_BONUS_DAYS", "int", "referral", "Приветственный бонус (дней)", min=0 ), SettingField("LEGACY_REFS", "bool", "referral", "Поддержка старых ref-ссылок"), + SettingField( + "MIGRATION_REMNASHOP_REFERRAL_CODE_COMPAT_ENABLED", + "bool", + "migrations", + "Старые ref-ссылки Remnashop", + "Принимать импортированные ref-коды Remnashop вместе с текущими кодами пользователей.", + subsection="Remnashop", + ), + SettingField( + "MIGRATION_REMNASHOP_PROMO_CODE_COMPAT_ENABLED", + "bool", + "migrations", + "Старые промокоды Remnashop", + "Пробовать точное совпадение промокода перед обычной uppercase-нормализацией.", + subsection="Remnashop", + ), + SettingField( + "MIGRATION_REMNASHOP_IMPORTED_AT", + "string", + "migrations", + "Последний импорт Remnashop", + "Заполняется скриптом импорта. Можно очистить, если отметка больше не нужна.", + subsection="Remnashop", + ), + SettingField( + "MIGRATION_REMNASHOP_NOTES", + "text", + "migrations", + "Заметки по миграции Remnashop", + "Внутренние заметки оператора по перенесенному инстансу.", + subsection="Remnashop", + ), # ─── Notifications ───────────────────────────────────────────── SettingField( "SUBSCRIPTION_NOTIFICATIONS_ENABLED", @@ -734,6 +766,7 @@ def manifest_payload() -> List[dict]: "devices": 10, "subscription_guides": 10, "system": 12, + "migrations": 13, } exclusive_map = { key: opposite diff --git a/backend/bot/app/web/webapp/auth.py b/backend/bot/app/web/webapp/auth.py index f1bfbe5..e1a7f2c 100644 --- a/backend/bot/app/web/webapp/auth.py +++ b/backend/bot/app/web/webapp/auth.py @@ -713,6 +713,7 @@ async def email_auth_verify_route(request: web.Request) -> web.Response: session, referral_param, current_user_id=None, + settings=settings, ) db_user, _ = await user_dal.create_email_user( session, @@ -821,6 +822,7 @@ async def email_auth_magic_route(request: web.Request) -> web.Response: session, referral_param, current_user_id=None, + settings=settings, ) db_user, _ = await user_dal.create_email_user( session, @@ -1292,17 +1294,35 @@ async def _link_telegram_to_user( return current_user -def _normalize_referral_param(raw: Optional[str]) -> Optional[str]: +def _remnashop_referral_compat_enabled(settings: Optional[Settings]) -> bool: + if settings is None: + return False + return bool(getattr(settings, "MIGRATION_REMNASHOP_REFERRAL_CODE_COMPAT_ENABLED", False)) + + +def _strip_referral_param_prefix( + raw: Optional[str], + *, + preserve_current_u_prefix: bool, +) -> str: value = (raw or "").strip() if not value: - return None + return "" value_lower = value.lower() - if value_lower.startswith("ref_u"): + if value_lower.startswith("ref_u") and not preserve_current_u_prefix: value = value[5:] elif value_lower.startswith("ref_"): value = value[4:] - elif value and value[0].lower() == "u" and len(value) == 10: + return value + + +def _normalize_referral_param(raw: Optional[str]) -> Optional[str]: + value = _strip_referral_param_prefix(raw, preserve_current_u_prefix=False) + if not value: + return None + + if value and value[0].lower() == "u" and len(value) == 10: value = value[1:] if not re.fullmatch(r"[A-Za-z0-9]{1,32}", value): @@ -1310,26 +1330,64 @@ def _normalize_referral_param(raw: Optional[str]) -> Optional[str]: return value.upper() +def _referral_param_lookup_candidates( + raw: Optional[str], + *, + remnashop_compat: bool, +) -> List[str]: + if not remnashop_compat: + normalized = _normalize_referral_param(raw) + return [normalized] if normalized else [] + + value = _strip_referral_param_prefix(raw, preserve_current_u_prefix=True) + if not value or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", value): + return [] + + candidates = [value] + if value and value[0].lower() == "u": + candidates.append(value[1:]) + + unique: List[str] = [] + for candidate in candidates: + if candidate and candidate not in unique: + unique.append(candidate) + return unique + + async def _resolve_referrer_id( session: AsyncSession, raw_referral_param: Optional[str], *, current_user_id: Optional[int], + settings: Optional[Settings] = None, ) -> Optional[int]: - normalized = _normalize_referral_param(raw_referral_param) - if not normalized: + remnashop_compat = _remnashop_referral_compat_enabled(settings) + candidates = _referral_param_lookup_candidates( + raw_referral_param, + remnashop_compat=remnashop_compat, + ) + if not candidates: return None - ref_user = None - if normalized.isdigit(): - ref_user = await user_dal.get_user_by_id(session, int(normalized)) - if not ref_user: - ref_user = await user_dal.get_user_by_referral_code(session, normalized) - if not ref_user: - return None - if current_user_id is not None and int(ref_user.user_id) == int(current_user_id): - return None - return int(ref_user.user_id) + for normalized in candidates: + ref_user = None + if normalized.isdigit() and not remnashop_compat: + ref_user = await user_dal.get_user_by_id(session, int(normalized)) + if not ref_user: + ref_user = await user_dal.get_user_by_referral_code( + session, + normalized, + include_legacy=remnashop_compat, + ) + if not ref_user and normalized.isdigit() and remnashop_compat: + ref_user = await user_dal.get_user_by_id(session, int(normalized)) + if not ref_user: + continue + if current_user_id is not None and int(ref_user.user_id) == int(current_user_id): + continue + return int(ref_user.user_id) + + return None async def _apply_referral_to_existing_user( @@ -1345,6 +1403,7 @@ async def _apply_referral_to_existing_user( session, raw_referral_param, current_user_id=int(user.user_id), + settings=request.app["settings"], ) if not referred_by_id: return False @@ -1427,6 +1486,7 @@ async def _ensure_user_from_telegram( session, referral_param or telegram_user.get("start_param"), current_user_id=user_id, + settings=settings, ) db_user, created = await user_dal.create_user( session, diff --git a/backend/bot/handlers/user/start.py b/backend/bot/handlers/user/start.py index 44cc598..79425a8 100644 --- a/backend/bot/handlers/user/start.py +++ b/backend/bot/handlers/user/start.py @@ -40,6 +40,67 @@ from db.models import User router = Router(name="user_start_router") +def _remnashop_referral_compat_enabled(settings: Settings) -> bool: + return bool(getattr(settings, "MIGRATION_REMNASHOP_REFERRAL_CODE_COMPAT_ENABLED", False)) + + +def _referral_code_lookup_candidates( + raw_ref_value: str, + *, + remnashop_compat: bool, +) -> list[str]: + value = str(raw_ref_value or "").strip() + if not value: + return [] + + candidates = [value] + if value and value[0].lower() == "u": + stripped_current_prefix = value[1:] + if remnashop_compat: + candidates.append(stripped_current_prefix) + else: + candidates = [stripped_current_prefix] + + unique: list[str] = [] + for candidate in candidates: + candidate = candidate.strip() + if candidate and candidate not in unique: + unique.append(candidate) + return unique + + +async def _resolve_referrer_from_start_ref( + session: AsyncSession, + raw_ref_value: str, + *, + settings: Settings, + current_user_id: int, +) -> Optional[int]: + ref_user: Optional[User] = None + if raw_ref_value.isdigit() and settings.LEGACY_REFS: + potential_referrer_id = int(raw_ref_value) + if potential_referrer_id != current_user_id: + ref_user = await user_dal.get_user_by_id(session, potential_referrer_id) + + include_legacy = _remnashop_referral_compat_enabled(settings) + if not ref_user: + for code in _referral_code_lookup_candidates( + raw_ref_value, + remnashop_compat=include_legacy, + ): + ref_user = await user_dal.get_user_by_referral_code( + session, + code, + include_legacy=include_legacy, + ) + if ref_user: + break + + if ref_user and ref_user.user_id != current_user_id: + return int(ref_user.user_id) + return None + + async def should_show_trial_button( settings: Settings, subscription_service: SubscriptionService, @@ -412,12 +473,10 @@ async def ensure_required_channel_subscription( @router.message(CommandStart()) @router.message( CommandStart( - magic=F.args.regexp(r"^ref_((?:[uU][A-Za-z0-9]{9})|(?:[A-Za-z0-9]{9})|\d+)$").as_( - "ref_match" - ) + magic=F.args.regexp(r"^ref_([A-Za-z0-9_-]{1,64})$").as_("ref_match") ) ) -@router.message(CommandStart(magic=F.args.regexp(r"^promo_(\w+)$").as_("promo_match"))) +@router.message(CommandStart(magic=F.args.regexp(r"^promo_([A-Za-z0-9_-]{1,100})$").as_("promo_match"))) @router.message(CommandStart(magic=F.args.regexp(r"^admin_user_(\d+)$").as_("admin_user_match"))) @router.message(CommandStart(magic=F.args.regexp(r"^ticket_(\d+)$").as_("ticket_match"))) @router.message(CommandStart(magic=F.args.regexp(r"^notifications$").as_("notifications_match"))) @@ -534,22 +593,12 @@ async def start_command_handler( if ref_match: raw_ref_value = ref_match.group(1) - if raw_ref_value.isdigit(): - if settings.LEGACY_REFS: - potential_referrer_id = int(raw_ref_value) - if potential_referrer_id != user_id and await user_dal.get_user_by_id( - session, potential_referrer_id - ): - referred_by_user_id = potential_referrer_id - else: - normalized_code = raw_ref_value.strip() - if normalized_code and normalized_code[0].lower() == "u": - normalized_code = normalized_code[1:] - ref_user = None - if normalized_code: - ref_user = await user_dal.get_user_by_referral_code(session, normalized_code) - if ref_user and ref_user.user_id != user_id: - referred_by_user_id = ref_user.user_id + referred_by_user_id = await _resolve_referrer_from_start_ref( + session, + raw_ref_value, + settings=settings, + current_user_id=user_id, + ) elif promo_match: promo_code_to_apply = promo_match.group(1) logging.info(f"User {user_id} started with promo code: {promo_code_to_apply}") diff --git a/backend/bot/services/promo_code_service.py b/backend/bot/services/promo_code_service.py index b486595..6f6a63d 100644 --- a/backend/bot/services/promo_code_service.py +++ b/backend/bot/services/promo_code_service.py @@ -38,8 +38,12 @@ class PromoCodeService: user_lang: str, ) -> Tuple[bool, datetime | str]: _ = lambda k, **kw: self.i18n.gettext(user_lang, k, **kw) - code_input_upper = (code_input or "").strip().upper()[:100] - code_display = html_escape(code_input_upper[:100], quote=False) + preserve_case = bool( + getattr(self.settings, "MIGRATION_REMNASHOP_PROMO_CODE_COMPAT_ENABLED", False) + ) + code_input_clean = (code_input or "").strip()[:100] + lookup_code = code_input_clean if preserve_case else code_input_clean.upper() + code_display = html_escape(lookup_code[:100], quote=False) throttle_identifier = self._throttle_identifier(user_id) throttle = await security_dal.check_throttle( @@ -54,7 +58,7 @@ class PromoCodeService: ) promo_data = await promo_code_dal.get_active_promo_code_by_code_str( - session, code_input_upper + session, lookup_code, preserve_case=preserve_case ) if not promo_data: @@ -71,9 +75,11 @@ class PromoCodeService: "promo_code_too_many_attempts", seconds=throttle_result.retry_after or max(1, int(self.settings.BRUTE_FORCE_LOCK_SECONDS)), - ) + ) return False, _("promo_code_not_found", code=code_display) + applied_code = str(promo_data.code or lookup_code) + code_display = html_escape(applied_code[:100], quote=False) existing_activation = await promo_code_dal.get_user_activation_for_promo( session, promo_data.promo_code_id, user_id ) @@ -86,7 +92,7 @@ class PromoCodeService: session=session, user_id=user_id, bonus_days=bonus_days, - reason=f"promo code {code_input_upper}", + reason=f"promo code {applied_code}", ) if new_end_date: @@ -109,7 +115,7 @@ class PromoCodeService: user = await user_dal.get_user_by_id(session, user_id) await notification_service.notify_promo_activation( user_id=user_id, - promo_code=code_input_upper, + promo_code=applied_code, bonus_days=bonus_days, username=user.username if user else None, email=getattr(user, "email", None) if user else None, diff --git a/backend/config/settings.py b/backend/config/settings.py index bd09406..3c856c7 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -286,6 +286,24 @@ class Settings(BaseSettings): default=True, description="Allow legacy referral links like ref_ to continue working. Defaults to True when unset.", # noqa: E501 ) + MIGRATION_REMNASHOP_REFERRAL_CODE_COMPAT_ENABLED: bool = Field( + default=False, + description=( + "Accept referral links imported from snoups/remnashop via legacy_referral_codes." + ), + ) + MIGRATION_REMNASHOP_PROMO_CODE_COMPAT_ENABLED: bool = Field( + default=False, + description="Try exact legacy Remnashop promo codes before uppercase normalization.", + ) + MIGRATION_REMNASHOP_IMPORTED_AT: Optional[str] = Field( + default=None, + description="Timestamp of the latest Remnashop import run, managed by the import script.", + ) + MIGRATION_REMNASHOP_NOTES: Optional[str] = Field( + default=None, + description="Operator notes for instances migrated from Remnashop.", + ) APP_RUNTIME_MODE: str = Field( default="production", diff --git a/backend/db/dal/promo_code_dal.py b/backend/db/dal/promo_code_dal.py index 8047792..41a376b 100644 --- a/backend/db/dal/promo_code_dal.py +++ b/backend/db/dal/promo_code_dal.py @@ -22,24 +22,46 @@ async def get_promo_code_by_id(session: AsyncSession, promo_code_id: int) -> Opt return await session.get(PromoCode, promo_code_id) -async def get_promo_code_by_code(session: AsyncSession, code_str: str) -> Optional[PromoCode]: +def _promo_lookup_candidates(code_str: str, *, preserve_case: bool) -> List[str]: + code = str(code_str or "").strip() + if not code: + return [] + candidates = [code] if preserve_case else [] + upper_code = code.upper() + if upper_code not in candidates: + candidates.append(upper_code) + return candidates + + +async def get_promo_code_by_code( + session: AsyncSession, code_str: str, *, preserve_case: bool = False +) -> Optional[PromoCode]: """Get promo code by code string (regardless of active status)""" - stmt = select(PromoCode).where(PromoCode.code == code_str.upper()) - result = await session.execute(stmt) - return result.scalar_one_or_none() + for candidate in _promo_lookup_candidates(code_str, preserve_case=preserve_case): + stmt = select(PromoCode).where(PromoCode.code == candidate) + result = await session.execute(stmt) + promo = result.scalar_one_or_none() + if promo: + return promo + return None async def get_active_promo_code_by_code_str( - session: AsyncSession, code_str: str + session: AsyncSession, code_str: str, *, preserve_case: bool = False ) -> Optional[PromoCode]: - stmt = select(PromoCode).where( - PromoCode.code == code_str.upper(), - PromoCode.is_active == True, - PromoCode.current_activations < PromoCode.max_activations, - or_(PromoCode.valid_until == None, PromoCode.valid_until > datetime.now(timezone.utc)), - ) - result = await session.execute(stmt) - return result.scalar_one_or_none() + now = datetime.now(timezone.utc) + for candidate in _promo_lookup_candidates(code_str, preserve_case=preserve_case): + stmt = select(PromoCode).where( + PromoCode.code == candidate, + PromoCode.is_active == True, + PromoCode.current_activations < PromoCode.max_activations, + or_(PromoCode.valid_until == None, PromoCode.valid_until > now), + ) + result = await session.execute(stmt) + promo = result.scalar_one_or_none() + if promo: + return promo + return None async def get_all_active_promo_codes( diff --git a/backend/db/dal/user_dal.py b/backend/db/dal/user_dal.py index bc58eef..0acc533 100644 --- a/backend/db/dal/user_dal.py +++ b/backend/db/dal/user_dal.py @@ -4,7 +4,7 @@ import string from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple -from sqlalchemy import and_, case, delete, desc, func, or_, update +from sqlalchemy import String, and_, case, cast, delete, desc, func, or_, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select @@ -14,6 +14,8 @@ from ..models import ( AdAttribution, EmailVerificationCode, HwidDevicePurchase, + LegacyImportMapping, + LegacyReferralCode, MessageLog, Payment, PromoCodeActivation, @@ -76,7 +78,7 @@ async def ensure_referral_code(session: AsyncSession, user: User) -> str: Returns the existing or newly generated code. """ if user.referral_code: - normalized = user.referral_code.strip().upper() + normalized = user.referral_code.strip() if normalized != user.referral_code: user.referral_code = normalized await session.flush() @@ -210,7 +212,7 @@ async def create_user(session: AsyncSession, user_data: Dict[str, Any]) -> Tuple if not user_data.get("referral_code"): user_data["referral_code"] = await generate_unique_referral_code(session) else: - user_data["referral_code"] = user_data["referral_code"].strip().upper() + user_data["referral_code"] = user_data["referral_code"].strip() # Use PostgreSQL upsert to avoid IntegrityError on concurrent inserts stmt = ( @@ -567,6 +569,19 @@ async def merge_users( await session.execute( update(model).where(model.user_id == source_user_id).values(user_id=target_user_id) ) + await session.execute( + update(LegacyReferralCode) + .where(LegacyReferralCode.user_id == source_user_id) + .values(user_id=target_user_id) + ) + await session.execute( + update(LegacyImportMapping) + .where( + LegacyImportMapping.target_table == "users", + LegacyImportMapping.target_id == str(source_user_id), + ) + .values(target_id=str(target_user_id)) + ) await session.execute( update(MessageLog) @@ -590,13 +605,60 @@ async def merge_users( return target -async def get_user_by_referral_code(session: AsyncSession, referral_code: str) -> Optional[User]: - normalized = referral_code.strip().upper() +async def get_user_by_referral_code( + session: AsyncSession, + referral_code: str, + *, + include_legacy: bool = False, +) -> Optional[User]: + normalized = referral_code.strip() if not normalized: return None + stmt = select(User).where(User.referral_code == normalized) result = await session.execute(stmt) - return result.scalar_one_or_none() + user = result.scalar_one_or_none() + if user: + return user + + upper_normalized = normalized.upper() + if upper_normalized != normalized: + stmt = select(User).where(User.referral_code == upper_normalized) + result = await session.execute(stmt) + user = result.scalar_one_or_none() + if user: + return user + + if not include_legacy: + return None + + stmt = ( + select(User) + .join(LegacyReferralCode, LegacyReferralCode.user_id == User.user_id) + .where(LegacyReferralCode.code == normalized, LegacyReferralCode.is_active == True) + .limit(1) + ) + result = await session.execute(stmt) + user = result.scalar_one_or_none() + if user: + return user + + if upper_normalized != normalized: + stmt = ( + select(User) + .join(LegacyReferralCode, LegacyReferralCode.user_id == User.user_id) + .where( + LegacyReferralCode.code == upper_normalized, + LegacyReferralCode.is_active == True, + ) + .limit(1) + ) + result = await session.execute(stmt) + user = result.scalar_one_or_none() + if user: + return user + + return None async def update_user( @@ -1020,6 +1082,31 @@ async def delete_user_and_relations(session: AsyncSession, user_id: int) -> bool await session.execute(delete(UserBilling).where(UserBilling.user_id == user_id)) await session.execute(delete(AdAttribution).where(AdAttribution.user_id == user_id)) await session.execute(delete(UserTelegramAvatar).where(UserTelegramAvatar.user_id == user_id)) + await session.execute(delete(LegacyReferralCode).where(LegacyReferralCode.user_id == user_id)) + await session.execute( + delete(LegacyImportMapping).where( + or_( + and_( + LegacyImportMapping.target_table == "users", + LegacyImportMapping.target_id == str(user_id), + ), + and_( + LegacyImportMapping.target_table == "subscriptions", + LegacyImportMapping.target_id.in_( + select(cast(Subscription.subscription_id, String)).where( + Subscription.user_id == user_id + ) + ), + ), + and_( + LegacyImportMapping.target_table == "payments", + LegacyImportMapping.target_id.in_( + select(cast(Payment.payment_id, String)).where(Payment.user_id == user_id) + ), + ), + ) + ) + ) await session.execute(delete(Payment).where(Payment.user_id == user_id)) await session.execute(delete(Subscription).where(Subscription.user_id == user_id)) diff --git a/backend/db/migrator.py b/backend/db/migrator.py index 73b3e3c..18a76fa 100644 --- a/backend/db/migrator.py +++ b/backend/db/migrator.py @@ -1070,6 +1070,79 @@ def _migration_0033_add_trial_eligibility_reset_marker(connection: Connection) - ) +def _migration_0034_add_legacy_import_compatibility(connection: Connection) -> None: + inspector = inspect(connection) + table_names = set(inspector.get_table_names()) + + if "users" in table_names: + columns = {col["name"]: col for col in inspector.get_columns("users")} + referral_column = columns.get("referral_code") + length = getattr(referral_column.get("type"), "length", None) if referral_column else None + if referral_column and (length is None or int(length) < 64): + connection.execute( + text("ALTER TABLE users ALTER COLUMN referral_code TYPE VARCHAR(64)") + ) + + connection.execute( + text( + """ + CREATE TABLE IF NOT EXISTS legacy_referral_codes ( + legacy_code_id SERIAL PRIMARY KEY, + source VARCHAR(64) NOT NULL DEFAULT 'remnashop', + code VARCHAR(128) NOT NULL, + user_id BIGINT NOT NULL REFERENCES users(user_id), + is_active BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NULL, + CONSTRAINT uq_legacy_referral_source_code UNIQUE (source, code) + ) + """ + ) + ) + for stmt in [ + ( + "CREATE INDEX IF NOT EXISTS ix_legacy_referral_codes_source " + "ON legacy_referral_codes (source)" + ), + "CREATE INDEX IF NOT EXISTS ix_legacy_referral_codes_code ON legacy_referral_codes (code)", + ( + "CREATE INDEX IF NOT EXISTS ix_legacy_referral_codes_user_id " + "ON legacy_referral_codes (user_id)" + ), + ( + "CREATE INDEX IF NOT EXISTS ix_legacy_referral_codes_is_active " + "ON legacy_referral_codes (is_active)" + ), + ]: + connection.execute(text(stmt)) + + connection.execute( + text( + """ + CREATE TABLE IF NOT EXISTS legacy_import_mappings ( + source VARCHAR(64) NOT NULL, + entity_type VARCHAR(64) NOT NULL, + source_id VARCHAR(128) NOT NULL, + target_table VARCHAR(128) NOT NULL, + target_id VARCHAR(128) NOT NULL, + metadata_json TEXT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NULL, + PRIMARY KEY (source, entity_type, source_id) + ) + """ + ) + ) + connection.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_legacy_import_mappings_target + ON legacy_import_mappings (target_table, target_id) + """ + ) + ) + + MIGRATIONS: List[Migration] = [ Migration( id="0001_add_channel_subscription_fields", @@ -1247,6 +1320,11 @@ MIGRATIONS: List[Migration] = [ description="Track admin resets of per-user trial eligibility without deleting history", upgrade=_migration_0033_add_trial_eligibility_reset_marker, ), + Migration( + id="0034_add_legacy_import_compatibility", + description="Store legacy import mappings and referral codes for source-bot migrations", + upgrade=_migration_0034_add_legacy_import_compatibility, + ), ] diff --git a/backend/db/models.py b/backend/db/models.py index 78bb8c3..e524d0d 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -43,7 +43,7 @@ class User(Base): registration_date = Column(DateTime(timezone=True), server_default=func.now()) is_banned = Column(Boolean, default=False) panel_user_uuid = Column(String, nullable=True, unique=True, index=True) - referral_code = Column(String(16), nullable=True, unique=True, index=True) + referral_code = Column(String(64), nullable=True, unique=True, index=True) referred_by_id = Column(BigInteger, ForeignKey("users.user_id"), nullable=True) lifetime_used_traffic_bytes = Column(BigInteger, nullable=True) lifetime_used_traffic_synced_at = Column(DateTime(timezone=True), nullable=True) @@ -396,6 +396,37 @@ class PromoCodeActivation(Base): ) +class LegacyReferralCode(Base): + __tablename__ = "legacy_referral_codes" + + legacy_code_id = Column(Integer, primary_key=True, autoincrement=True) + source = Column(String(64), nullable=False, default="remnashop", index=True) + code = Column(String(128), nullable=False, index=True) + user_id = Column(BigInteger, ForeignKey("users.user_id"), nullable=False, index=True) + is_active = Column(Boolean, nullable=False, default=True, index=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + user = relationship("User") + + __table_args__ = ( + UniqueConstraint("source", "code", name="uq_legacy_referral_source_code"), + ) + + +class LegacyImportMapping(Base): + __tablename__ = "legacy_import_mappings" + + source = Column(String(64), primary_key=True) + entity_type = Column(String(64), primary_key=True) + source_id = Column(String(128), primary_key=True) + target_table = Column(String(128), nullable=False) + target_id = Column(String(128), nullable=False) + metadata_json = Column(Text, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), onupdate=func.now(), nullable=True) + + class MessageLog(Base): __tablename__ = "message_logs" diff --git a/backend/scripts/__init__.py b/backend/scripts/__init__.py new file mode 100644 index 0000000..7e7e6c0 --- /dev/null +++ b/backend/scripts/__init__.py @@ -0,0 +1,2 @@ +"""Operational one-shot scripts shipped with the backend image.""" + diff --git a/backend/scripts/import_legacy.py b/backend/scripts/import_legacy.py new file mode 100644 index 0000000..b2fede7 --- /dev/null +++ b/backend/scripts/import_legacy.py @@ -0,0 +1,1164 @@ +"""Import data from legacy source bots into the current shop database. + +Currently supported source: + remnashop + +Example: + python backend/scripts/import_legacy.py \ + --source-type remnashop \ + --source-dsn postgresql://user:pass@localhost:5432/remnashop \ + --dry-run +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import re +import sys +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Iterable, Optional + +from sqlalchemy import inspect, select, text +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import ( + AsyncConnection, + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +BACKEND_ROOT = Path(__file__).resolve().parents[1] +if str(BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(BACKEND_ROOT)) + +from config.settings import Settings # noqa: E402 +from db.dal import user_dal # noqa: E402 +from db.migrator import run_database_migrations # noqa: E402 +from db.models import ( # noqa: E402 + AppSettingOverride, + Base, + LegacyImportMapping, + LegacyReferralCode, + MessageLog, + Payment, + PromoCode, + PromoCodeActivation, + Subscription, + User, +) + +SOURCE = "remnashop" +GIB = 1024**3 +UUID_RE = re.compile( + r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" +) +SAFE_SCHEMA_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +logger = logging.getLogger(__name__) + + +def normalize_async_postgres_dsn(dsn: str) -> str: + value = str(dsn or "").strip() + if value.startswith("postgresql+asyncpg://"): + return value + if value.startswith("postgresql://"): + return "postgresql+asyncpg://" + value.removeprefix("postgresql://") + if value.startswith("postgres://"): + return "postgresql+asyncpg://" + value.removeprefix("postgres://") + return value + + +def _json_default(value: Any) -> str: + if isinstance(value, (datetime, Decimal)): + return str(value) + return str(value) + + +def _json_dumps(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, default=_json_default) + + +def _safe_schema_name(schema: str) -> str: + value = str(schema or "public").strip() + if not SAFE_SCHEMA_RE.fullmatch(value): + raise ValueError(f"Unsafe PostgreSQL schema name: {schema!r}") + return value + + +def _qtable(schema: str, table: str) -> str: + schema = _safe_schema_name(schema) + return f'"{schema}"."{table}"' + + +def _as_mapping(row: Any) -> dict[str, Any]: + return dict(row._mapping if hasattr(row, "_mapping") else row) + + +def _as_utc(value: Any) -> Optional[datetime]: + if value is None: + return None + if isinstance(value, datetime): + result = value + else: + text_value = str(value).strip() + if not text_value: + return None + try: + result = datetime.fromisoformat(text_value.replace("Z", "+00:00")) + except ValueError: + return None + if result.tzinfo is None: + return result.replace(tzinfo=timezone.utc) + return result.astimezone(timezone.utc) + + +def _to_decimal(value: Any) -> Optional[Decimal]: + if value is None: + return None + try: + return Decimal(str(value)) + except (InvalidOperation, ValueError): + return None + + +def _to_int(value: Any) -> Optional[int]: + number = _to_decimal(value) + if number is None: + return None + try: + return int(number) + except (OverflowError, ValueError): + return None + + +def _split_name(name: Any) -> tuple[Optional[str], Optional[str]]: + value = str(name or "").strip() + if not value: + return None, None + parts = value.split(maxsplit=1) + if len(parts) == 1: + return parts[0][:255], None + return parts[0][:255], parts[1][:255] + + +def _jsonish(value: Any) -> dict[str, Any]: + if isinstance(value, dict): + return value + if isinstance(value, str) and value.strip(): + try: + decoded = json.loads(value) + except ValueError: + return {} + return decoded if isinstance(decoded, dict) else {} + return {} + + +def _listish(value: Any) -> list[Any]: + if value is None: + return [] + if isinstance(value, list): + return value + if isinstance(value, tuple): + return list(value) + return [value] + + +def remnashop_traffic_gb_to_bytes(value: Any) -> Optional[int]: + number = _to_decimal(value) + if number is None: + return None + return int(number * GIB) + + +def remnashop_pricing_amount(pricing: Any) -> float: + data = _jsonish(pricing) + for key in ("final_amount", "total_amount", "amount", "price"): + number = _to_decimal(data.get(key)) + if number is not None: + return float(number) + return 0.0 + + +def remnashop_pricing_currency(pricing: Any, fallback: Any = None) -> str: + data = _jsonish(pricing) + currency = str(data.get("currency") or fallback or "RUB").strip().upper() + return currency or "RUB" + + +def remnashop_transaction_status(status: Any, gateway_type: Any = None) -> str: + source_status = str(status or "").strip().upper() + provider = str(gateway_type or "").strip().lower() + if source_status == "COMPLETED": + return "succeeded" + if source_status == "PENDING": + return f"pending_{provider}" if provider else "pending" + if source_status == "CANCELED": + return "canceled" + if source_status == "REFUNDED": + return "refunded" + if source_status == "FAILED": + return "failed" + return source_status.lower() or "unknown" + + +def remnashop_sale_mode(purchase_type: Any) -> str: + source_type = str(purchase_type or "").strip().upper() + if source_type in {"NEW", "RENEW"}: + return "subscription" + if source_type == "CHANGE": + return "tariff_upgrade" + return source_type.lower() or "subscription" + + +def remnashop_months_from_plan_snapshot( + plan_snapshot: Any, + *, + created_at: Any = None, + expire_at: Any = None, +) -> Optional[int]: + data = _jsonish(plan_snapshot) + for key in ("duration_months", "months", "month"): + months = _to_int(data.get(key)) + if months and months > 0: + return months + + for key in ("duration_days", "days", "duration"): + days = _to_int(data.get(key)) + if days and days > 0: + return max(1, round(days / 30)) + + start = _as_utc(created_at) + end = _as_utc(expire_at) + if start and end and end > start: + return max(1, round((end - start).days / 30)) + return None + + +def remnashop_tariff_key(plan_snapshot: Any, tariff_map: dict[str, str]) -> Optional[str]: + data = _jsonish(plan_snapshot) + candidates = [ + data.get("id"), + data.get("name"), + data.get("tag"), + data.get("public_code"), + ] + for candidate in candidates: + key = str(candidate or "").strip() + if key and key in tariff_map: + return tariff_map[key] + return None + + +def _provider_value(gateway_type: Any) -> str: + value = str(gateway_type or "remnashop").strip().lower() + if value == "telegram_stars": + return "stars" + return value or "remnashop" + + +def _extract_panel_subscription_uuid(url: Any, panel_user_uuid: Optional[str]) -> Optional[str]: + value = str(url or "") + if not value: + return None + panel_user_uuid = str(panel_user_uuid or "").lower() + for match in UUID_RE.finditer(value): + candidate = match.group(0).lower() + if candidate != panel_user_uuid: + return candidate + return None + + +def _legacy_user_metadata(row: dict[str, Any]) -> dict[str, Any]: + keys = ( + "id", + "points", + "personal_discount", + "purchase_discount", + "role", + "is_rules_accepted", + "is_trial_available", + "language", + "current_subscription_id", + ) + return {key: row.get(key) for key in keys if row.get(key) is not None} + + +def _counter() -> dict[str, int]: + return defaultdict(int) + + +class RemnashopImporter: + def __init__( + self, + *, + source: AsyncConnection, + target: AsyncSession, + source_schema: str, + only: set[str], + on_conflict: str, + dry_run: bool, + created_by_admin_id: int, + tariff_map: dict[str, str], + write_admin_compat_overrides: bool, + ) -> None: + self.source = source + self.target = target + self.source_schema = _safe_schema_name(source_schema) + self.only = only + self.on_conflict = on_conflict + self.dry_run = dry_run + self.created_by_admin_id = created_by_admin_id + self.tariff_map = tariff_map + self.write_admin_compat_overrides = write_admin_compat_overrides + self.tables: set[str] = set() + self.user_map: dict[int, int] = {} + self.summary: dict[str, Any] = { + "source": SOURCE, + "dry_run": dry_run, + "on_conflict": on_conflict, + "users": _counter(), + "referrals": _counter(), + "subscriptions": _counter(), + "payments": _counter(), + "promocodes": _counter(), + "settings": _counter(), + "warnings": [], + } + + async def run(self) -> dict[str, Any]: + self.tables = await self._source_tables() + await self._warn_missing_tables() + + if self._should_run("users"): + await self.import_users() + if self._should_run("referrals"): + await self.import_referrals() + if self._should_run("subscriptions"): + await self.import_subscriptions() + if self._should_run("payments"): + await self.import_payments() + if self._should_run("promocodes"): + await self.import_promocodes() + if self._should_run("settings"): + await self.import_settings() + + if self.write_admin_compat_overrides: + await self._write_admin_overrides() + + return self._plain_summary() + + def _plain_summary(self) -> dict[str, Any]: + result = dict(self.summary) + for key, value in list(result.items()): + if isinstance(value, defaultdict): + result[key] = dict(value) + return result + + def _should_run(self, key: str) -> bool: + return not self.only or key in self.only or "all" in self.only + + async def _source_tables(self) -> set[str]: + def load_tables(sync_connection: Any) -> set[str]: + return set(inspect(sync_connection).get_table_names(schema=self.source_schema)) + + return await self.source.run_sync(load_tables) + + async def _warn_missing_tables(self) -> None: + required = {"users", "subscriptions", "transactions", "referrals", "settings"} + missing = sorted(required - self.tables) + if missing: + self.summary["warnings"].append(f"Missing source tables: {', '.join(missing)}") + + async def _fetch_rows(self, table: str, *, order_by: str = "id") -> list[dict[str, Any]]: + if table not in self.tables: + return [] + order_sql = f" ORDER BY {order_by}" if order_by else "" + result = await self.source.execute( + text(f"SELECT * FROM {_qtable(self.source_schema, table)}{order_sql}") + ) + return [_as_mapping(row) for row in result.mappings().all()] + + async def _fetch_one(self, table: str) -> Optional[dict[str, Any]]: + rows = await self._fetch_rows(table, order_by="") + return rows[0] if rows else None + + async def _latest_panel_uuid_by_telegram(self) -> dict[int, str]: + if "subscriptions" not in self.tables: + return {} + result = await self.source.execute( + text( + f""" + SELECT DISTINCT ON (user_telegram_id) + user_telegram_id, + user_remna_id + FROM {_qtable(self.source_schema, "subscriptions")} + WHERE user_remna_id IS NOT NULL + ORDER BY user_telegram_id, updated_at DESC NULLS LAST, id DESC + """ + ) + ) + panel_by_tg: dict[int, str] = {} + for row in result.mappings().all(): + telegram_id = _to_int(row.get("user_telegram_id")) + panel_uuid = str(row.get("user_remna_id") or "").strip() + if telegram_id and panel_uuid: + panel_by_tg[telegram_id] = panel_uuid + return panel_by_tg + + async def _target_user_for_telegram(self, telegram_id: Any) -> Optional[User]: + normalized = _to_int(telegram_id) + if normalized is None: + return None + user = await user_dal.get_user_by_telegram_id(self.target, normalized) + if not user: + user = await user_dal.get_user_by_id(self.target, normalized) + if user: + self.user_map[normalized] = int(user.user_id) + return user + + def _can_overwrite(self) -> bool: + return self.on_conflict == "overwrite" + + def _can_merge_existing(self) -> bool: + return self.on_conflict in {"merge", "overwrite"} + + def _assign_if_allowed(self, model: Any, attr: str, value: Any) -> bool: + if value is None: + return False + current = getattr(model, attr, None) + if self._can_overwrite() or current in (None, ""): + setattr(model, attr, value) + return True + return False + + async def _upsert_mapping( + self, + *, + entity_type: str, + source_id: Any, + target_table: str, + target_id: Any, + metadata: Optional[dict[str, Any]] = None, + ) -> None: + now = datetime.now(timezone.utc) + source_id_value = str(source_id) + target_id_value = str(target_id) + stmt = ( + pg_insert(LegacyImportMapping) + .values( + source=SOURCE, + entity_type=entity_type, + source_id=source_id_value, + target_table=target_table, + target_id=target_id_value, + metadata_json=_json_dumps(metadata or {}), + updated_at=now, + ) + .on_conflict_do_update( + index_elements=[ + LegacyImportMapping.source, + LegacyImportMapping.entity_type, + LegacyImportMapping.source_id, + ], + set_={ + "target_table": target_table, + "target_id": target_id_value, + "metadata_json": _json_dumps(metadata or {}), + "updated_at": now, + }, + ) + ) + await self.target.execute(stmt) + + async def _get_mapping(self, entity_type: str, source_id: Any) -> Optional[LegacyImportMapping]: + stmt = select(LegacyImportMapping).where( + LegacyImportMapping.source == SOURCE, + LegacyImportMapping.entity_type == entity_type, + LegacyImportMapping.source_id == str(source_id), + ) + result = await self.target.execute(stmt) + return result.scalar_one_or_none() + + async def _upsert_setting_override(self, key: str, value: Any) -> None: + now = datetime.now(timezone.utc) + encoded = json.dumps(value, ensure_ascii=False, separators=(",", ":")) + stmt = ( + pg_insert(AppSettingOverride) + .values( + key=key, + value=encoded, + updated_at=now, + updated_by=self.created_by_admin_id or None, + ) + .on_conflict_do_update( + index_elements=[AppSettingOverride.key], + set_={ + "value": encoded, + "updated_at": now, + "updated_by": self.created_by_admin_id or None, + }, + ) + ) + await self.target.execute(stmt) + + async def _upsert_legacy_referral_code(self, *, code: str, user_id: int) -> None: + if len(code) > 128: + self.summary["warnings"].append( + f"Skipped overlong legacy referral code for user {user_id}: {len(code)} chars" + ) + return + now = datetime.now(timezone.utc) + stmt = ( + pg_insert(LegacyReferralCode) + .values( + source=SOURCE, + code=code, + user_id=user_id, + is_active=True, + updated_at=now, + ) + .on_conflict_do_update( + index_elements=[LegacyReferralCode.source, LegacyReferralCode.code], + set_={"user_id": user_id, "is_active": True, "updated_at": now}, + ) + ) + await self.target.execute(stmt) + + async def _record_user_state_note( + self, + *, + telegram_id: int, + user_id: int, + metadata: dict[str, Any], + ) -> None: + if not metadata: + return + if await self._get_mapping("user_state", telegram_id): + return + log = MessageLog( + user_id=None, + target_user_id=user_id, + event_type="legacy_remnashop_user_state", + content=_json_dumps(metadata), + is_admin_event=True, + ) + self.target.add(log) + await self.target.flush() + await self._upsert_mapping( + entity_type="user_state", + source_id=telegram_id, + target_table="message_logs", + target_id=log.log_id, + metadata=metadata, + ) + + async def _source_referral_code_conflicts(self, code: str, user_id: int) -> bool: + existing = await user_dal.get_user_by_referral_code( + self.target, + code, + include_legacy=False, + ) + return bool(existing and int(existing.user_id) != int(user_id)) + + async def import_users(self) -> None: + rows = await self._fetch_rows("users", order_by="telegram_id") + panel_by_tg = await self._latest_panel_uuid_by_telegram() + for row in rows: + telegram_id = _to_int(row.get("telegram_id")) + if telegram_id is None: + self.summary["users"]["skipped"] += 1 + continue + + first_name, last_name = _split_name(row.get("name")) + panel_uuid = panel_by_tg.get(telegram_id) + referral_code = str(row.get("referral_code") or "").strip() or None + created_at = _as_utc(row.get("created_at")) or datetime.now(timezone.utc) + language = str(row.get("language") or "ru").strip().lower()[:8] or "ru" + + existing = await self._target_user_for_telegram(telegram_id) + if existing and self.on_conflict == "skip": + target = existing + self.summary["users"]["skipped"] += 1 + elif existing: + target = existing + if self._can_merge_existing(): + self._assign_if_allowed(target, "username", row.get("username")) + self._assign_if_allowed(target, "first_name", first_name) + self._assign_if_allowed(target, "last_name", last_name) + self._assign_if_allowed(target, "language_code", language) + self._assign_if_allowed(target, "panel_user_uuid", panel_uuid) + if bool(row.get("is_blocked")): + target.is_banned = True + elif self._can_overwrite(): + target.is_banned = False + if bool(row.get("is_bot_blocked")): + target.telegram_notifications_status = "blocked" + target.telegram_notifications_checked_at = datetime.now(timezone.utc) + target.telegram_notifications_blocked_at = datetime.now(timezone.utc) + if referral_code and len(referral_code) <= 64 and not target.referral_code: + if not await self._source_referral_code_conflicts( + referral_code, + int(target.user_id), + ): + target.referral_code = referral_code + self.summary["users"]["updated"] += 1 + else: + new_referral_code = None + if referral_code and len(referral_code) <= 64: + conflict = await self._source_referral_code_conflicts( + referral_code, + telegram_id, + ) + if not conflict: + new_referral_code = referral_code + + target, created = await user_dal.create_user( + self.target, + { + "user_id": telegram_id, + "telegram_id": telegram_id, + "username": row.get("username"), + "first_name": first_name, + "last_name": last_name, + "language_code": language, + "registration_date": created_at, + "is_banned": bool(row.get("is_blocked")), + "panel_user_uuid": panel_uuid, + "referral_code": new_referral_code, + "telegram_notifications_status": "blocked" + if bool(row.get("is_bot_blocked")) + else "unknown", + "telegram_notifications_checked_at": datetime.now(timezone.utc) + if bool(row.get("is_bot_blocked")) + else None, + "telegram_notifications_blocked_at": datetime.now(timezone.utc) + if bool(row.get("is_bot_blocked")) + else None, + }, + ) + self.summary["users"]["created" if created else "updated"] += 1 + + if not target: + self.summary["users"]["skipped"] += 1 + continue + + self.user_map[telegram_id] = int(target.user_id) + if referral_code: + await self._upsert_legacy_referral_code(code=referral_code, user_id=target.user_id) + + metadata = _legacy_user_metadata(row) + if panel_uuid: + metadata["panel_user_uuid"] = panel_uuid + await self._upsert_mapping( + entity_type="user", + source_id=telegram_id, + target_table="users", + target_id=target.user_id, + metadata=metadata, + ) + await self._record_user_state_note( + telegram_id=telegram_id, + user_id=int(target.user_id), + metadata=metadata, + ) + + await self.target.flush() + + async def import_referrals(self) -> None: + rows = await self._fetch_rows("referrals", order_by="id") + for row in rows: + referrer = await self._target_user_for_telegram(row.get("referrer_telegram_id")) + referred = await self._target_user_for_telegram(row.get("referred_telegram_id")) + if not referrer or not referred or referrer.user_id == referred.user_id: + self.summary["referrals"]["skipped"] += 1 + continue + if referred.referred_by_id and not self._can_overwrite(): + self.summary["referrals"]["skipped"] += 1 + continue + referred.referred_by_id = int(referrer.user_id) + self.summary["referrals"]["updated"] += 1 + await self._upsert_mapping( + entity_type="referral", + source_id=row.get("id") or f"{referrer.user_id}:{referred.user_id}", + target_table="users", + target_id=referred.user_id, + metadata={ + "referrer_user_id": referrer.user_id, + "referred_user_id": referred.user_id, + }, + ) + await self.target.flush() + + async def import_subscriptions(self) -> None: + rows = await self._fetch_rows("subscriptions", order_by="id") + now = datetime.now(timezone.utc) + for row in rows: + user = await self._target_user_for_telegram(row.get("user_telegram_id")) + if not user: + self.summary["subscriptions"]["skipped"] += 1 + continue + + panel_user_uuid = str(row.get("user_remna_id") or user.panel_user_uuid or "").strip() + if not panel_user_uuid: + self.summary["subscriptions"]["skipped"] += 1 + continue + if not user.panel_user_uuid or self._can_overwrite(): + user.panel_user_uuid = panel_user_uuid + + source_id = row.get("id") + mapping = await self._get_mapping("subscription", source_id) + existing: Optional[Subscription] = None + if mapping and str(mapping.target_id).isdigit(): + existing = await self.target.get(Subscription, int(mapping.target_id)) + + panel_sub_uuid = _extract_panel_subscription_uuid(row.get("url"), panel_user_uuid) + if not existing and panel_sub_uuid: + existing = ( + await self.target.execute( + select(Subscription).where( + Subscription.panel_subscription_uuid == panel_sub_uuid + ) + ) + ).scalar_one_or_none() + + status = str(row.get("status") or "UNKNOWN").strip().upper() + expire_at = _as_utc(row.get("expire_at")) or now + created_at = _as_utc(row.get("created_at")) or now + plan_snapshot = _jsonish(row.get("plan_snapshot")) + traffic_limit_bytes = remnashop_traffic_gb_to_bytes(row.get("traffic_limit")) + payload = { + "user_id": int(user.user_id), + "panel_user_uuid": panel_user_uuid, + "panel_subscription_uuid": panel_sub_uuid, + "start_date": created_at, + "end_date": expire_at, + "duration_months": remnashop_months_from_plan_snapshot( + plan_snapshot, + created_at=created_at, + expire_at=expire_at, + ), + "is_active": status in {"ACTIVE", "LIMITED"} and expire_at > now, + "status_from_panel": status, + "traffic_limit_bytes": traffic_limit_bytes, + "provider": "trial" if bool(row.get("is_trial")) else SOURCE, + "skip_notifications": True, + "auto_renew_enabled": False, + "tariff_key": remnashop_tariff_key(plan_snapshot, self.tariff_map), + "tier_baseline_bytes": traffic_limit_bytes, + "period_start_at": created_at, + "hwid_device_limit": _to_int(row.get("device_limit")), + } + metadata = { + "source": SOURCE, + "source_subscription_id": source_id, + "traffic_limit_strategy": str(row.get("traffic_limit_strategy") or ""), + "tag": row.get("tag"), + "internal_squads": [str(item) for item in _listish(row.get("internal_squads"))], + "external_squad": str(row.get("external_squad") or "") or None, + "url": row.get("url"), + "plan_snapshot": plan_snapshot, + } + + if existing: + if self.on_conflict == "skip": + self.summary["subscriptions"]["skipped"] += 1 + else: + for key, value in payload.items(): + self._assign_if_allowed(existing, key, value) + self.summary["subscriptions"]["updated"] += 1 + target_subscription_id = existing.subscription_id + else: + subscription = Subscription(**payload) + self.target.add(subscription) + await self.target.flush() + target_subscription_id = subscription.subscription_id + self.summary["subscriptions"]["created"] += 1 + + await self._upsert_mapping( + entity_type="subscription", + source_id=source_id, + target_table="subscriptions", + target_id=target_subscription_id, + metadata=metadata, + ) + + await self.target.flush() + + async def import_payments(self) -> None: + rows = await self._fetch_rows("transactions", order_by="id") + for row in rows: + user = await self._target_user_for_telegram(row.get("user_telegram_id")) + if not user: + self.summary["payments"]["skipped"] += 1 + continue + + provider_payment_id = f"{SOURCE}:{row.get('payment_id') or row.get('id')}" + existing = ( + await self.target.execute( + select(Payment).where(Payment.provider_payment_id == provider_payment_id) + ) + ).scalar_one_or_none() + + provider = _provider_value(row.get("gateway_type")) + plan_snapshot = _jsonish(row.get("plan_snapshot")) + created_at = _as_utc(row.get("created_at")) + payload = { + "user_id": int(user.user_id), + "provider_payment_id": provider_payment_id, + "provider": provider, + "amount": remnashop_pricing_amount(row.get("pricing")), + "currency": remnashop_pricing_currency(row.get("pricing"), row.get("currency")), + "status": remnashop_transaction_status(row.get("status"), provider), + "description": self._payment_description(row), + "subscription_duration_months": remnashop_months_from_plan_snapshot( + plan_snapshot, + created_at=row.get("created_at"), + expire_at=None, + ), + "sale_mode": remnashop_sale_mode(row.get("purchase_type")), + "tariff_key": remnashop_tariff_key(plan_snapshot, self.tariff_map), + "created_at": created_at, + } + payload = {key: value for key, value in payload.items() if value is not None} + + if existing: + if self.on_conflict == "skip": + self.summary["payments"]["skipped"] += 1 + else: + for key, value in payload.items(): + self._assign_if_allowed(existing, key, value) + self.summary["payments"]["updated"] += 1 + target_payment_id = existing.payment_id + else: + payment = Payment(**payload) + self.target.add(payment) + await self.target.flush() + target_payment_id = payment.payment_id + self.summary["payments"]["created"] += 1 + + await self._upsert_mapping( + entity_type="payment", + source_id=row.get("payment_id") or row.get("id"), + target_table="payments", + target_id=target_payment_id, + metadata={ + "source_transaction_id": row.get("id"), + "is_test": row.get("is_test"), + "purchase_type": str(row.get("purchase_type") or ""), + "gateway_type": str(row.get("gateway_type") or ""), + "plan_snapshot": plan_snapshot, + }, + ) + + await self.target.flush() + + def _payment_description(self, row: dict[str, Any]) -> str: + snapshot = _jsonish(row.get("plan_snapshot")) + plan_name = str(snapshot.get("name") or snapshot.get("tag") or "").strip() + purchase_type = str(row.get("purchase_type") or "").strip().upper() + if plan_name: + return f"Remnashop import: {purchase_type} {plan_name}".strip() + return f"Remnashop import: {purchase_type}".strip() + + async def import_promocodes(self) -> None: + if "promocodes" not in self.tables: + self.summary["promocodes"]["missing_source_table"] += 1 + return + + activation_rows_by_code = await self._source_promocode_activation_rows() + rows = await self._fetch_rows("promocodes", order_by="id") + for row in rows: + code = str(row.get("code") or "").strip() + if not code: + self.summary["promocodes"]["skipped"] += 1 + continue + + bonus_days = self._promo_bonus_days(row) + if bonus_days is None or bonus_days <= 0: + self.summary["promocodes"]["unsupported_reward"] += 1 + continue + + existing = ( + await self.target.execute(select(PromoCode).where(PromoCode.code == code)) + ).scalar_one_or_none() + activations = activation_rows_by_code.get(code, []) + valid_until = None + lifetime_days = _to_int(row.get("lifetime")) + if lifetime_days and _as_utc(row.get("created_at")): + valid_until = _as_utc(row.get("created_at")) + if valid_until: + valid_until = valid_until + timedelta(days=lifetime_days) + + payload = { + "code": code, + "bonus_days": int(bonus_days), + "max_activations": _to_int(row.get("max_activations")) or 1_000_000, + "current_activations": len(activations), + "is_active": bool(row.get("is_active")), + "created_by_admin_id": self.created_by_admin_id, + "created_at": _as_utc(row.get("created_at")), + "valid_until": valid_until, + } + payload = {key: value for key, value in payload.items() if value is not None} + + if existing: + if self.on_conflict == "skip": + self.summary["promocodes"]["skipped"] += 1 + else: + for key, value in payload.items(): + self._assign_if_allowed(existing, key, value) + self.summary["promocodes"]["updated"] += 1 + promo = existing + else: + promo = PromoCode(**payload) + self.target.add(promo) + await self.target.flush() + self.summary["promocodes"]["created"] += 1 + + await self._upsert_mapping( + entity_type="promocode", + source_id=row.get("id") or code, + target_table="promo_codes", + target_id=promo.promo_code_id, + metadata={ + "reward_type": str(row.get("reward_type") or ""), + "reward": row.get("reward"), + "plan": _jsonish(row.get("plan")), + "lifetime": row.get("lifetime"), + }, + ) + await self._import_promocode_activations(promo, activations) + + await self.target.flush() + + async def _source_promocode_activation_rows(self) -> dict[str, list[dict[str, Any]]]: + if "promocode_activations" not in self.tables: + return {} + result = await self.source.execute( + text( + f""" + SELECT a.*, p.code + FROM {_qtable(self.source_schema, "promocode_activations")} a + JOIN {_qtable(self.source_schema, "promocodes")} p + ON p.id = a.promocode_id + ORDER BY a.id + """ + ) + ) + by_code: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in result.mappings().all(): + mapping = _as_mapping(row) + code = str(mapping.get("code") or "").strip() + if code: + by_code[code].append(mapping) + return by_code + + def _promo_bonus_days(self, row: dict[str, Any]) -> Optional[int]: + reward_type = str(row.get("reward_type") or "").strip().upper() + if reward_type == "DURATION": + return _to_int(row.get("reward")) + if reward_type == "SUBSCRIPTION": + plan = _jsonish(row.get("plan")) + return ( + _to_int(plan.get("duration_days")) + or _to_int(plan.get("days")) + or _to_int(row.get("reward")) + ) + return None + + async def _import_promocode_activations( + self, + promo: PromoCode, + activations: Iterable[dict[str, Any]], + ) -> None: + for activation in activations: + user = await self._target_user_for_telegram(activation.get("user_telegram_id")) + if not user: + self.summary["promocodes"]["activation_skipped"] += 1 + continue + stmt = ( + pg_insert(PromoCodeActivation) + .values( + promo_code_id=promo.promo_code_id, + user_id=user.user_id, + activated_at=_as_utc(activation.get("activated_at")) + or datetime.now(timezone.utc), + ) + .on_conflict_do_nothing( + index_elements=[ + PromoCodeActivation.promo_code_id, + PromoCodeActivation.user_id, + ] + ) + ) + await self.target.execute(stmt) + self.summary["promocodes"]["activation_imported"] += 1 + + async def import_settings(self) -> None: + source_settings = await self._fetch_one("settings") + plans = ( + await self._fetch_rows("plans", order_by="order_index") + if "plans" in self.tables + else [] + ) + notes = { + "default_currency": ( + source_settings.get("default_currency") if source_settings else None + ), + "settings": { + key: source_settings.get(key) + for key in ("access", "requirements", "notifications", "referral", "menu") + if source_settings and source_settings.get(key) is not None + }, + "plans_count": len(plans), + "plans": [ + { + "id": plan.get("id"), + "name": plan.get("name"), + "type": str(plan.get("type") or ""), + "traffic_limit": plan.get("traffic_limit"), + "device_limit": plan.get("device_limit"), + "tag": plan.get("tag"), + } + for plan in plans[:100] + ], + } + await self._upsert_mapping( + entity_type="settings", + source_id="singleton", + target_table="app_setting_overrides", + target_id="MIGRATION_REMNASHOP_NOTES", + metadata=notes, + ) + self.summary["settings"]["captured"] += 1 + + async def _write_admin_overrides(self) -> None: + now = datetime.now(timezone.utc).isoformat() + plain_summary = self._plain_summary() + await self._upsert_setting_override( + "MIGRATION_REMNASHOP_REFERRAL_CODE_COMPAT_ENABLED", + True, + ) + await self._upsert_setting_override( + "MIGRATION_REMNASHOP_PROMO_CODE_COMPAT_ENABLED", + "promocodes" in self.tables, + ) + await self._upsert_setting_override("MIGRATION_REMNASHOP_IMPORTED_AT", now) + await self._upsert_setting_override( + "MIGRATION_REMNASHOP_NOTES", + _json_dumps(plain_summary), + ) + self.summary["settings"]["admin_overrides_written"] += 1 + + +def parse_only(value: str) -> set[str]: + if not value: + return set() + return {item.strip().lower() for item in value.split(",") if item.strip()} + + +def parse_tariff_map(value: Optional[str]) -> dict[str, str]: + if not value: + return {} + path = Path(value) + raw = path.read_text(encoding="utf-8") if path.exists() else value + decoded = json.loads(raw) + if not isinstance(decoded, dict): + raise ValueError("--tariff-map-json must be a JSON object or a path to one") + return {str(key): str(mapped) for key, mapped in decoded.items()} + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Import legacy bot data into this shop.") + parser.add_argument("--source-type", choices=[SOURCE], default=SOURCE) + parser.add_argument("--source-dsn", required=True) + parser.add_argument("--source-schema", default="public") + parser.add_argument("--target-dsn") + parser.add_argument( + "--only", + default="all", + help=( + "Comma-separated sections: " + "all,users,referrals,subscriptions,payments,promocodes,settings" + ), + ) + parser.add_argument( + "--on-conflict", + choices=["merge", "skip", "overwrite"], + default="merge", + ) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--created-by-admin-id", type=int, default=0) + parser.add_argument( + "--tariff-map-json", + help="JSON object or path mapping remnashop plan id/name/tag to local tariff_key.", + ) + parser.add_argument( + "--no-admin-compat-overrides", + action="store_true", + help="Do not enable migration compatibility toggles in admin settings.", + ) + return parser + + +async def _prepare_target_schema(engine: Any) -> None: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + await connection.run_sync(run_database_migrations) + + +async def run_import(args: argparse.Namespace) -> dict[str, Any]: + settings = Settings() + source_engine = create_async_engine(normalize_async_postgres_dsn(args.source_dsn)) + target_engine = create_async_engine( + normalize_async_postgres_dsn(args.target_dsn or settings.DATABASE_URL) + ) + await _prepare_target_schema(target_engine) + + session_factory = async_sessionmaker( + bind=target_engine, + class_=AsyncSession, + expire_on_commit=False, + autocommit=False, + autoflush=False, + ) + + async with source_engine.connect() as source, session_factory() as target: + importer = RemnashopImporter( + source=source, + target=target, + source_schema=args.source_schema, + only=parse_only(args.only), + on_conflict=args.on_conflict, + dry_run=bool(args.dry_run), + created_by_admin_id=args.created_by_admin_id, + tariff_map=parse_tariff_map(args.tariff_map_json), + write_admin_compat_overrides=not args.no_admin_compat_overrides, + ) + summary = await importer.run() + if args.dry_run: + await target.rollback() + else: + await target.commit() + + await source_engine.dispose() + await target_engine.dispose() + return summary + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + args = build_arg_parser().parse_args() + summary = asyncio.run(run_import(args)) + print(_json_dumps(summary)) + + +if __name__ == "__main__": + main() diff --git a/frontend/src/admin/sections/SettingsSection.svelte b/frontend/src/admin/sections/SettingsSection.svelte index 3443016..8d151fb 100644 --- a/frontend/src/admin/sections/SettingsSection.svelte +++ b/frontend/src/admin/sections/SettingsSection.svelte @@ -348,6 +348,7 @@ devices: "Устройства", subscription_guides: "Connection guides", system: "Система", + migrations: "Миграции", }; return adminText(`settings_section_${id}`, {}, map[id] || id); } diff --git a/locales/en.json b/locales/en.json index cf3ffff..c6ae75d 100644 --- a/locales/en.json +++ b/locales/en.json @@ -1108,11 +1108,13 @@ "admin_settings_section_devices": "Devices", "admin_settings_section_support": "Support", "admin_settings_section_system": "System", + "admin_settings_section_migrations": "Migrations", "admin_settings_field_telemetry_enabled_label": "Anonymous install analytics", "admin_settings_field_telemetry_enabled_description": "Sends one anonymous heartbeat per day (version, OS, locale, user-count range). No personal data, tokens or domains. Helps gauge how many installs are active and which versions are in use. Toggling this off takes effect without a restart.", "admin_settings_subsection_common": "Common", "admin_settings_subsection_checkout": "Checkout", "admin_settings_subsection_remnawave": "Remnawave", + "admin_settings_subsection_remnashop": "Remnashop", "admin_settings_subsection_telegram_stars": "Telegram Stars", "admin_settings_subsection_yookassa": "YooKassa", "admin_settings_subsection_freekassa": "FreeKassa", @@ -1127,6 +1129,14 @@ "admin_settings_provider_webhook_base_missing": "Set WEBHOOK_BASE_URL in .env to show the full URL for {path}.", "admin_settings_provider_admin_only_label": "Only for admins", "admin_settings_provider_admin_only_description": "Shows this provider only to admins. Webhooks and payment status handling remain active for test payments.", + "admin_settings_field_migration_remnashop_referral_code_compat_enabled_label": "Remnashop ref-code compatibility", + "admin_settings_field_migration_remnashop_referral_code_compat_enabled_description": "Allows old Remnashop referral codes to resolve without changing their case or format. The importer enables it automatically.", + "admin_settings_field_migration_remnashop_promo_code_compat_enabled_label": "Remnashop promo-code compatibility", + "admin_settings_field_migration_remnashop_promo_code_compat_enabled_description": "Looks up promo codes in the original case first, then falls back to the current rules. Regular promo codes keep working as before.", + "admin_settings_field_migration_remnashop_imported_at_label": "Remnashop import date", + "admin_settings_field_migration_remnashop_imported_at_description": "Operational marker for the latest Remnashop import, stored as an ISO timestamp.", + "admin_settings_field_migration_remnashop_notes_label": "Remnashop import notes", + "admin_settings_field_migration_remnashop_notes_description": "Short importer summary: migrated entities and enabled compatibility modes.", "admin_settings_validation_errors": "Errors: {errors}", "admin_settings_save_error": "Error: {error}", "admin_sync_started": "Synchronization started", diff --git a/locales/ru.json b/locales/ru.json index 4ce0660..e7d5047 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -1108,11 +1108,13 @@ "admin_settings_section_devices": "Устройства", "admin_settings_section_support": "Поддержка", "admin_settings_section_system": "Система", + "admin_settings_section_migrations": "Миграции", "admin_settings_field_telemetry_enabled_label": "Анонимная статистика установки", "admin_settings_field_telemetry_enabled_description": "Раз в сутки отправляет обезличенный сигнал: версия, ОС, локаль и число пользователей в виде диапазона. Без персональных данных, токенов и доменов. Помогает оценить число активных установок и используемые версии. Отключение применяется без перезапуска.", "admin_settings_subsection_common": "Общие", "admin_settings_subsection_checkout": "Оформление оплаты", "admin_settings_subsection_remnawave": "Remnawave", + "admin_settings_subsection_remnashop": "Remnashop", "admin_settings_subsection_telegram_stars": "Telegram Stars", "admin_settings_subsection_yookassa": "YooKassa", "admin_settings_subsection_freekassa": "FreeKassa", @@ -1127,6 +1129,14 @@ "admin_settings_provider_webhook_base_missing": "Укажите WEBHOOK_BASE_URL в .env, чтобы увидеть полный адрес для {path}.", "admin_settings_provider_admin_only_label": "Только для админов", "admin_settings_provider_admin_only_description": "Показывает провайдер только администраторам. Вебхуки и обработка статусов остаются активными для тестовых платежей.", + "admin_settings_field_migration_remnashop_referral_code_compat_enabled_label": "Совместимость ref-кодов Remnashop", + "admin_settings_field_migration_remnashop_referral_code_compat_enabled_description": "Разрешает вход по старым ref-кодам Remnashop без изменения их регистра и формата. Включается импортёром автоматически.", + "admin_settings_field_migration_remnashop_promo_code_compat_enabled_label": "Совместимость промокодов Remnashop", + "admin_settings_field_migration_remnashop_promo_code_compat_enabled_description": "Ищет промокоды сначала в исходном регистре, затем по текущим правилам. Обычные промокоды продолжают работать как раньше.", + "admin_settings_field_migration_remnashop_imported_at_label": "Дата импорта Remnashop", + "admin_settings_field_migration_remnashop_imported_at_description": "Служебная отметка последнего импорта Remnashop в ISO-формате.", + "admin_settings_field_migration_remnashop_notes_label": "Заметки импорта Remnashop", + "admin_settings_field_migration_remnashop_notes_description": "Краткая сводка импортёра: какие сущности перенесены и какие совместимые режимы включены.", "admin_settings_validation_errors": "Ошибки: {errors}", "admin_settings_save_error": "Ошибка: {error}", "admin_sync_started": "Синхронизация запущена", diff --git a/tests/test_admin_settings_manifest_i18n.py b/tests/test_admin_settings_manifest_i18n.py index 2144100..21ecbed 100644 --- a/tests/test_admin_settings_manifest_i18n.py +++ b/tests/test_admin_settings_manifest_i18n.py @@ -44,6 +44,13 @@ BACKUP_SETTINGS = ( "BACKUP_COMPOSE_ENABLED", ) +REMNASHOP_MIGRATION_SETTINGS = ( + "MIGRATION_REMNASHOP_REFERRAL_CODE_COMPAT_ENABLED", + "MIGRATION_REMNASHOP_PROMO_CODE_COMPAT_ENABLED", + "MIGRATION_REMNASHOP_IMPORTED_AT", + "MIGRATION_REMNASHOP_NOTES", +) + ADMIN_TARIFF_SETTINGS_PAGE_KEYS = { "admin_tariffs_trial_title", "admin_tariffs_trial_subtitle", @@ -204,6 +211,27 @@ def test_backup_settings_i18n_keys_exist(): assert field["i18n_description_key"] in messages +def test_remnashop_migration_settings_i18n_keys_exist(): + manifest = _manifest_by_key() + + for setting_key in REMNASHOP_MIGRATION_SETTINGS: + field = manifest[setting_key] + assert field["section"] == "migrations" + assert field["section_order"] == 13 + assert field["subsection"] == "Remnashop" + assert field["i18n_subsection_key"] == "admin_settings_subsection_remnashop" + + for language in ("ru", "en"): + messages = _locale(language) + + assert "admin_settings_section_migrations" in messages + assert "admin_settings_subsection_remnashop" in messages + for setting_key in REMNASHOP_MIGRATION_SETTINGS: + field = manifest[setting_key] + assert field["i18n_label_key"] in messages + assert field["i18n_description_key"] in messages + + def test_backup_required_numeric_settings_reject_empty_values(): with pytest.raises(ValueError): coerce_value(get_field_by_key("BACKUP_INTERVAL_SECONDS"), "") diff --git a/tests/test_remnashop_import.py b/tests/test_remnashop_import.py new file mode 100644 index 0000000..fb4a16b --- /dev/null +++ b/tests/test_remnashop_import.py @@ -0,0 +1,44 @@ +from datetime import datetime, timezone + +from scripts.import_legacy import ( + remnashop_months_from_plan_snapshot, + remnashop_pricing_amount, + remnashop_pricing_currency, + remnashop_sale_mode, + remnashop_traffic_gb_to_bytes, + remnashop_transaction_status, +) + + +def test_remnashop_pricing_helpers_read_final_amount_and_currency(): + pricing = {"final_amount": "199.50", "currency": "rub"} + + assert remnashop_pricing_amount(pricing) == 199.5 + assert remnashop_pricing_currency(pricing) == "RUB" + + +def test_remnashop_traffic_limit_is_converted_from_gib(): + assert remnashop_traffic_gb_to_bytes(10) == 10 * 1024**3 + assert remnashop_traffic_gb_to_bytes(None) is None + + +def test_remnashop_status_and_sale_mode_mapping_matches_current_payment_model(): + assert remnashop_transaction_status("COMPLETED", "YOOKASSA") == "succeeded" + assert remnashop_transaction_status("PENDING", "WATA") == "pending_wata" + assert remnashop_transaction_status("CANCELED", "WATA") == "canceled" + assert remnashop_sale_mode("NEW") == "subscription" + assert remnashop_sale_mode("RENEW") == "subscription" + assert remnashop_sale_mode("CHANGE") == "tariff_upgrade" + + +def test_remnashop_plan_months_prefers_snapshot_then_dates(): + assert remnashop_months_from_plan_snapshot({"duration_days": 90}) == 3 + assert remnashop_months_from_plan_snapshot({"months": 12}) == 12 + assert ( + remnashop_months_from_plan_snapshot( + {}, + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + expire_at=datetime(2026, 4, 1, tzinfo=timezone.utc), + ) + == 3 + ) diff --git a/tests/test_support_migration.py b/tests/test_support_migration.py index 5fa5292..2e5fed5 100644 --- a/tests/test_support_migration.py +++ b/tests/test_support_migration.py @@ -1,5 +1,11 @@ from db.migrator import MIGRATIONS -from db.models import SupportTicket, SupportTicketMessage, User +from db.models import ( + LegacyImportMapping, + LegacyReferralCode, + SupportTicket, + SupportTicketMessage, + User, +) def test_support_migration_is_registered_after_existing_revisions(): @@ -39,3 +45,18 @@ def test_trial_eligibility_reset_migration_and_model_are_registered(): "0032_add_telegram_notification_status" ) assert "trial_eligibility_reset_at" in User.__table__.columns + + +def test_legacy_import_compatibility_migration_and_models_are_registered(): + ids = [migration.id for migration in MIGRATIONS] + + assert "0034_add_legacy_import_compatibility" in ids + assert ids.index("0034_add_legacy_import_compatibility") > ids.index( + "0033_add_trial_eligibility_reset_marker" + ) + assert User.__table__.columns["referral_code"].type.length == 64 + assert LegacyReferralCode.__tablename__ == "legacy_referral_codes" + assert LegacyImportMapping.__tablename__ == "legacy_import_mappings" + assert "uq_legacy_referral_source_code" in { + constraint.name for constraint in LegacyReferralCode.__table__.constraints + } diff --git a/tests/test_user_dal.py b/tests/test_user_dal.py index 5b8bd30..8849ba9 100644 --- a/tests/test_user_dal.py +++ b/tests/test_user_dal.py @@ -230,6 +230,8 @@ class UserDalMergeTests(unittest.IsolatedAsyncioTestCase): ) self.assertIn("support_ticket_messages", update_tables) self.assertIn("email_verification_codes", delete_tables) + self.assertIn("legacy_referral_codes", delete_tables) + self.assertIn("legacy_import_mappings", delete_tables) session.delete.assert_awaited_once_with(user) session.flush.assert_awaited_once() @@ -366,6 +368,8 @@ class UserDalMergeTests(unittest.IsolatedAsyncioTestCase): self.assertIn("payments", update_tables) self.assertIn("promo_code_activations", update_tables) self.assertIn("user_payment_methods", update_tables) + self.assertIn("legacy_referral_codes", update_tables) + self.assertIn("legacy_import_mappings", update_tables) self.assertIn("message_logs", update_tables) self.assertIn("users", update_tables) self.assertIn("user_payment_methods", delete_tables) From d22ed238b4f8ad9b89f6defd486df9195b8ddf52 Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 10:08:59 +0300 Subject: [PATCH 02/11] feat: add shell install wizard --- docs/getting-started/deployment.md | 31 + docs/migrations/index.md | 1 + docs/migrations/remnashop.md | 60 ++ scripts/install.sh | 902 +++++++++++++++++++++++++++++ tests/test_install_script.py | 39 ++ 5 files changed, 1033 insertions(+) create mode 100644 docs/migrations/remnashop.md create mode 100644 scripts/install.sh create mode 100644 tests/test_install_script.py diff --git a/docs/getting-started/deployment.md b/docs/getting-started/deployment.md index cee2da1..ae3f09d 100644 --- a/docs/getting-started/deployment.md +++ b/docs/getting-started/deployment.md @@ -13,6 +13,37 @@ docker compose ps docker compose logs -f backend worker frontend ``` +## Интерактивный install wizard + +Для нового сервера можно не клонировать весь репозиторий и не ставить Python. +Скачайте только POSIX `sh`-скрипт через `raw.githubusercontent.com` и +запустите его: + +```bash +curl -fsSL https://raw.githubusercontent.com/3252a8/remnawave-minishop/main/scripts/install.sh -o install.sh +sh install.sh +``` + +Wizard работает через меню с цифрами и подтверждениями `y/n`. Он умеет: + +- скачать выбранный compose-профиль (`Caddy`, `Nginx`, `Pangolin/Newt` или `no-proxy`); +- сгенерировать минимальный `.env`, включая пароли и стабильные secrets; +- сохранить backup существующих файлов перед перезаписью; +- запустить `docker compose pull && docker compose up -d`; +- проверить текущий стек через `docker compose ps` и логи `migrate`; +- запустить миграцию из поддерживаемых legacy-ботов, сейчас - из Remnashop. + +Для тестирования другой ветки или форка задайте источник перед запуском: + +```bash +MINISHOP_INSTALL_REPO=3252a8/remnawave-minishop \ +MINISHOP_INSTALL_REF=main \ +sh install.sh +``` + +Миграция в wizard сначала запускает `dry-run`, показывает JSON-сводку и только +после отдельного подтверждения применяет изменения в целевую БД. + Обычный `docker compose up -d --build` поднимает: - `postgres` и `redis` с проверками здоровья; diff --git a/docs/migrations/index.md b/docs/migrations/index.md index 501a37a..0c20758 100644 --- a/docs/migrations/index.md +++ b/docs/migrations/index.md @@ -5,3 +5,4 @@ | Источник | Поддерживаемый случай | Документы | | --- | --- | --- | | [remnawave-tg-shop](https://github.com/kavore/remnawave-tg-shop/) | Полный перенос всех данных | [Инструкция](remnawave-tg-shop.md) | +| [Remnashop](https://github.com/snoups/remnashop/) | Автоматический импорт пользователей, подписок, платежей, рефералов и промокодов | [Инструкция](remnashop.md) | diff --git a/docs/migrations/remnashop.md b/docs/migrations/remnashop.md new file mode 100644 index 0000000..2387bb9 --- /dev/null +++ b/docs/migrations/remnashop.md @@ -0,0 +1,60 @@ +# Миграция из Remnashop + +Remnashop импортируется через общий legacy-importer `backend/scripts/import_legacy.py`. +Самый удобный путь - интерактивный install wizard: + +```bash +curl -fsSL https://raw.githubusercontent.com/3252a8/remnawave-minishop/main/scripts/install.sh -o install.sh +sh install.sh +``` + +В меню выберите `Install new stack and run legacy migration` для нового сервера +или `Run legacy migration only`, если compose-папка и `.env` уже готовы. + +## Что переносится + +- пользователи Telegram, username, email, Remnawave UUID и метаданные профиля; +- старые referral codes и связи рефералов; +- подписки, сроки, лимиты трафика, HWID/device limit и UUID подписок панели; +- платежи и статусы платежей; +- промокоды на дни подписки и их активации, если таблицы есть в source DB; +- служебные mappings, чтобы повторный запуск мог работать в режиме `merge`; +- настройки совместимости Remnashop в админке: старые ref-ссылки и promo codes. + +Данные, которые не имеют прямого аналога, сохраняются в legacy mappings или +message logs как заметки, чтобы администратор мог проверить их после переноса. + +## Flow wizard + +1. Wizard скачивает compose-профиль и `backend/scripts/import_legacy.py` через + `raw.githubusercontent.com`, без клонирования репозитория. +2. Вы указываете source PostgreSQL DSN Remnashop и schema, обычно `public`. +3. Вы выбираете целевую БД: текущую compose-БД или ручной target DSN. +4. При необходимости указываете JSON map тарифов Remnashop в локальные + `tariff_key`, например `{"basic": "standard_month"}`. +5. Wizard запускает `dry-run` и показывает JSON-сводку. +6. После подтверждения `y` importer применяет изменения и перезапускает + `backend`/`worker`, чтобы настройки совместимости перечитались. + +Если source DB находится на том же Docker host, помните, что DSN выполняется +из backend-контейнера. Для подключения к сервису вне compose-сети может +понадобиться host name вроде `host.docker.internal`, внешний адрес сервера или +ручное подключение контейнеров к общей Docker network. + +## Ручной запуск + +Если нужно запустить importer без wizard: + +```bash +docker compose run --rm backend \ + python backend/scripts/import_legacy.py \ + --source-type remnashop \ + --source-dsn 'postgresql://old_user:old_password@old_host:5432/remnashop' \ + --source-schema public \ + --dry-run +``` + +После успешного `dry-run` повторите команду без `--dry-run`. По умолчанию +режим конфликтов `merge`: существующие пользователи и платежи сопоставляются, +а новые записи добавляются. Для узкого импорта используйте `--only`, например +`--only users,referrals,promocodes`. diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100644 index 0000000..d9d93dc --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,902 @@ +#!/bin/sh +set -u + +# Dependency-light installer for fresh Linux hosts. +# It intentionally avoids Python and clones only the files selected by the user. + +DEFAULT_REPO="${MINISHOP_INSTALL_REPO:-3252a8/remnawave-minishop}" +DEFAULT_REF="${MINISHOP_INSTALL_REF:-main}" +DEFAULT_IMAGE_TAG="${MINISHOP_IMAGE_TAG:-latest}" +INSTALL_STATE_DIR=".installer" +IMPORTER_CACHE_PATH="$INSTALL_STATE_DIR/import_legacy.py" +APP_UID=10001 +APP_GID=10001 + +if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then + RESET="$(printf '\033[0m')" + BOLD="$(printf '\033[1m')" + DIM="$(printf '\033[2m')" + RED="$(printf '\033[31m')" + GREEN="$(printf '\033[32m')" + YELLOW="$(printf '\033[33m')" + BLUE="$(printf '\033[34m')" + CYAN="$(printf '\033[36m')" +else + RESET="" + BOLD="" + DIM="" + RED="" + GREEN="" + YELLOW="" + BLUE="" + CYAN="" +fi + +TARGET_DIR="" +SOURCE_REPO="" +SOURCE_REF="" +PROFILE_KEY="" +ENV_PATH="" +COMPOSE_STYLE="" +PROMPT_VALUE="" +CHOICE_VALUE="" + +COMPOSE_PROJECT_NAME_VALUE="" +IMAGE_TAG_VALUE="" +WEBHOOK_HOST_VALUE="" +MINIAPP_HOST_VALUE="" +WEBHOOK_PUBLIC_URL_VALUE="" +MINIAPP_PUBLIC_URL_VALUE="" +HTTP_BIND_VALUE="" +HTTPS_BIND_VALUE="" +WEB_SERVER_BIND_VALUE="" +FRONTEND_BIND_VALUE="" +PANGOLIN_ENDPOINT_VALUE="" +NEWT_ID_VALUE="" +NEWT_SECRET_VALUE="" +BOT_TOKEN_VALUE="" +ADMIN_IDS_VALUE="" +POSTGRES_USER_VALUE="" +POSTGRES_PASSWORD_VALUE="" +POSTGRES_DB_VALUE="" +WEBAPP_ENABLED_VALUE="" +WEBAPP_SESSION_SECRET_VALUE="" +WEBHOOK_SECRET_TOKEN_VALUE="" +TRUSTED_PROXIES_VALUE="" +PANEL_API_URL_VALUE="" +PANEL_API_KEY_VALUE="" +PANEL_WEBHOOK_SECRET_VALUE="" + +KNOWN_ENV_KEYS="COMPOSE_PROJECT_NAME IMAGE_TAG WEBHOOK_HOST MINIAPP_HOST WEBHOOK_PUBLIC_URL MINIAPP_PUBLIC_URL HTTP_BIND HTTPS_BIND WEB_SERVER_BIND FRONTEND_BIND PANGOLIN_ENDPOINT NEWT_ID NEWT_SECRET BOT_TOKEN ADMIN_IDS POSTGRES_USER POSTGRES_PASSWORD POSTGRES_DB WEBAPP_ENABLED WEBAPP_SESSION_SECRET WEBHOOK_SECRET_TOKEN TRUSTED_PROXIES PANEL_API_URL PANEL_API_KEY PANEL_WEBHOOK_SECRET" + +color() { + printf '%s%s%s' "$2" "$1" "$RESET" +} + +banner() { + printf '\n' + color "Remnawave MiniShop Install Wizard" "$BOLD$CYAN" + printf '\n' + color "Install, configure, start, and migrate legacy bot data." "$DIM" + printf '\n\n' +} + +section() { + printf '\n' + color "== $1 ==" "$BOLD$BLUE" + printf '\n' +} + +info() { + color "* " "$CYAN" + printf '%s\n' "$1" +} + +warn() { + color "! " "$YELLOW" + printf '%s\n' "$1" +} + +ok() { + color "[ok] " "$GREEN" + printf '%s\n' "$1" +} + +fail() { + color "[x] " "$RED" + printf '%s\n' "$1" >&2 +} + +pause() { + printf '%s' "${DIM}Press Enter to continue...${RESET}" + # shellcheck disable=SC2034 + read -r _ +} + +print_help() { + cat </dev/null | tail -n 1 || true) + if [ -n "$line" ]; then + strip_quotes "${line#*=}" + return 0 + fi + fi + printf '%s' "$default_value" +} + +known_env_key() { + case " $KNOWN_ENV_KEYS " in + *" $1 "*) + return 0 + ;; + *) + return 1 + ;; + esac +} + +secret_hex() { + bytes="${1:-32}" + if command -v openssl >/dev/null 2>&1; then + openssl rand -hex "$bytes" + return 0 + fi + if [ -r /dev/urandom ] && command -v od >/dev/null 2>&1; then + dd if=/dev/urandom bs="$bytes" count=1 2>/dev/null | od -An -tx1 | tr -d ' \n' + return 0 + fi + fail "Could not generate a secure secret. Install openssl and retry." + exit 1 +} + +generated_password() { + secret_hex 24 +} + +raw_url() { + repo=$(printf '%s' "$1" | sed 's#^/*##; s#/*$##') + ref=$(printf '%s' "$2" | sed 's#^/*##; s#/*$##') + path=$(printf '%s' "$3" | sed 's#^/*##') + printf 'https://raw.githubusercontent.com/%s/%s/%s' "$repo" "$ref" "$path" +} + +download_to() { + url="$1" + target="$2" + if command -v curl >/dev/null 2>&1; then + curl -fsSL "$url" -o "$target" + return $? + fi + if command -v wget >/dev/null 2>&1; then + wget -qO "$target" "$url" + return $? + fi + fail "curl or wget is required to download files." + exit 1 +} + +backup_path() { + path="$1" + stamp=$(date -u '+%Y%m%d-%H%M%S') + printf '%s.bak-%s' "$path" "$stamp" +} + +write_downloaded_file() { + source_path="$1" + target_path="$2" + mkdir -p "$(dirname "$target_path")" + if [ -e "$target_path" ]; then + if confirm "$target_path exists. Overwrite with backup?" 0; then + backup=$(backup_path "$target_path") + cp "$target_path" "$backup" + info "Backed up $target_path to $(basename "$backup")" + else + warn "Keeping existing $target_path" + rm -f "$source_path" + return 0 + fi + fi + mv "$source_path" "$target_path" + ok "Wrote $target_path" +} + +download_raw_file() { + source="$1" + target="$2" + required="${3:-1}" + url=$(raw_url "$SOURCE_REPO" "$SOURCE_REF" "$source") + tmp="$TARGET_DIR/.download.$$.$(basename "$target")" + if download_to "$url" "$tmp"; then + write_downloaded_file "$tmp" "$TARGET_DIR/$target" + return 0 + fi + rm -f "$tmp" + if [ "$required" = "1" ]; then + fail "Could not download $url" + return 1 + fi + warn "Skipping optional file $source" + return 0 +} + +choose_profile() { + choose "Deployment profile" "1" "1|2|3|4" \ + "1. Caddy HTTPS - recommended public HTTPS with automatic certificates." \ + "2. Nginx HTTPS - TLS certificates are managed manually." \ + "3. Pangolin / Newt - no inbound ports; public routes are configured in Pangolin." \ + "4. No proxy / external TLS - direct HTTP ports or an external TLS terminator." + case "$CHOICE_VALUE" in + 1) PROFILE_KEY="caddy" ;; + 2) PROFILE_KEY="nginx" ;; + 3) PROFILE_KEY="newt" ;; + 4) PROFILE_KEY="no-proxy" ;; + esac +} + +download_profile_files() { + section "Download deployment files" + case "$PROFILE_KEY" in + caddy) + download_raw_file "deploy/examples/caddy/docker-compose.yml" "docker-compose.yml" 1 || return 1 + download_raw_file "deploy/examples/caddy/Caddyfile" "Caddyfile" 1 || return 1 + download_raw_file "deploy/examples/caddy/.env.example" ".env.example" 1 || return 1 + ;; + nginx) + download_raw_file "deploy/examples/nginx/docker-compose.yml" "docker-compose.yml" 1 || return 1 + download_raw_file "deploy/examples/nginx/nginx.conf.template" "nginx.conf.template" 1 || return 1 + download_raw_file "deploy/examples/nginx/.env.example" ".env.example" 1 || return 1 + download_raw_file "deploy/examples/nginx/ssl/README.md" "ssl/README.md" 1 || return 1 + ;; + newt) + download_raw_file "deploy/examples/newt/docker-compose.yml" "docker-compose.yml" 1 || return 1 + download_raw_file "deploy/examples/newt/.env.example" ".env.example" 1 || return 1 + ;; + no-proxy) + download_raw_file "deploy/examples/no-proxy/docker-compose.yml" "docker-compose.yml" 1 || return 1 + download_raw_file "deploy/examples/no-proxy/.env.example" ".env.example" 1 || return 1 + ;; + esac +} + +prompt_common_env() { + section "Minimal .env" + prompt_value "Compose project name" "$(env_get COMPOSE_PROJECT_NAME remnawave-minishop)" 0 0 "" + COMPOSE_PROJECT_NAME_VALUE="$PROMPT_VALUE" + prompt_value "Image tag" "$(env_get IMAGE_TAG "$DEFAULT_IMAGE_TAG")" 0 0 "" + IMAGE_TAG_VALUE="$PROMPT_VALUE" + prompt_value "Telegram bot token" "$(env_get BOT_TOKEN '')" 1 1 "" + BOT_TOKEN_VALUE="$PROMPT_VALUE" + prompt_value "Admin Telegram IDs, comma-separated" "$(env_get ADMIN_IDS '')" 1 0 "" + ADMIN_IDS_VALUE="$PROMPT_VALUE" + prompt_value "Postgres user" "$(env_get POSTGRES_USER remnawave_minishop)" 1 0 "" + POSTGRES_USER_VALUE="$PROMPT_VALUE" + existing_postgres_password=$(env_get POSTGRES_PASSWORD "") + if [ -z "$existing_postgres_password" ]; then + existing_postgres_password=$(generated_password) + fi + prompt_value "Postgres password" "$existing_postgres_password" 1 1 "" + POSTGRES_PASSWORD_VALUE="$PROMPT_VALUE" + prompt_value "Postgres database" "$(env_get POSTGRES_DB remnawave_minishop)" 1 0 "" + POSTGRES_DB_VALUE="$PROMPT_VALUE" + + WEBAPP_ENABLED_VALUE="$(env_get WEBAPP_ENABLED True)" + WEBAPP_SESSION_SECRET_VALUE="$(env_get WEBAPP_SESSION_SECRET "")" + if [ -z "$WEBAPP_SESSION_SECRET_VALUE" ]; then + WEBAPP_SESSION_SECRET_VALUE="$(secret_hex 32)" + fi + WEBHOOK_SECRET_TOKEN_VALUE="$(env_get WEBHOOK_SECRET_TOKEN "")" + if [ -z "$WEBHOOK_SECRET_TOKEN_VALUE" ]; then + WEBHOOK_SECRET_TOKEN_VALUE="$(secret_hex 32)" + fi + + prompt_value "Remnawave Panel API URL" "$(env_get PANEL_API_URL https://panel.example.com/api)" 0 0 "url" + PANEL_API_URL_VALUE="$PROMPT_VALUE" + prompt_value "Remnawave Panel API key" "$(env_get PANEL_API_KEY change_me)" 0 1 "" + PANEL_API_KEY_VALUE="$PROMPT_VALUE" + existing_panel_webhook_secret=$(env_get PANEL_WEBHOOK_SECRET "") + if [ -z "$existing_panel_webhook_secret" ]; then + existing_panel_webhook_secret=$(secret_hex 24) + fi + prompt_value "Remnawave Panel webhook secret" "$existing_panel_webhook_secret" 0 1 "" + PANEL_WEBHOOK_SECRET_VALUE="$PROMPT_VALUE" + + case "$PROFILE_KEY" in + caddy|nginx|newt) + prompt_value "Webhook/API public hostname" "$(env_get WEBHOOK_HOST webhooks.example.com)" 1 0 "hostname" + WEBHOOK_HOST_VALUE="$PROMPT_VALUE" + prompt_value "Mini App public hostname" "$(env_get MINIAPP_HOST app.example.com)" 1 0 "hostname" + MINIAPP_HOST_VALUE="$PROMPT_VALUE" + TRUSTED_PROXIES_VALUE="$(env_get TRUSTED_PROXIES '127.0.0.1,::1,172.16.0.0/12')" + ;; + esac + + case "$PROFILE_KEY" in + caddy|nginx) + prompt_value "HTTP bind" "$(env_get HTTP_BIND '0.0.0.0:80')" 0 0 "" + HTTP_BIND_VALUE="$PROMPT_VALUE" + prompt_value "HTTPS bind" "$(env_get HTTPS_BIND '0.0.0.0:443')" 0 0 "" + HTTPS_BIND_VALUE="$PROMPT_VALUE" + ;; + newt) + prompt_value "Pangolin endpoint" "$(env_get PANGOLIN_ENDPOINT https://pangolin.example.com)" 1 0 "url" + PANGOLIN_ENDPOINT_VALUE="$PROMPT_VALUE" + prompt_value "Newt ID" "$(env_get NEWT_ID '')" 1 0 "" + NEWT_ID_VALUE="$PROMPT_VALUE" + prompt_value "Newt secret" "$(env_get NEWT_SECRET '')" 1 1 "" + NEWT_SECRET_VALUE="$PROMPT_VALUE" + ;; + no-proxy) + prompt_value "Backend bind" "$(env_get WEB_SERVER_BIND '0.0.0.0:8080')" 0 0 "" + WEB_SERVER_BIND_VALUE="$PROMPT_VALUE" + prompt_value "Frontend bind" "$(env_get FRONTEND_BIND '0.0.0.0:8082')" 0 0 "" + FRONTEND_BIND_VALUE="$PROMPT_VALUE" + prompt_value "Webhook public URL" "$(env_get WEBHOOK_PUBLIC_URL 'http://127.0.0.1:8080')" 1 0 "url" + WEBHOOK_PUBLIC_URL_VALUE="$PROMPT_VALUE" + prompt_value "Mini App public URL" "$(env_get MINIAPP_PUBLIC_URL 'http://127.0.0.1:8082/')" 1 0 "url" + MINIAPP_PUBLIC_URL_VALUE="$PROMPT_VALUE" + TRUSTED_PROXIES_VALUE="$(env_get TRUSTED_PROXIES '127.0.0.1,::1')" + ;; + esac +} + +env_line() { + key="$1" + value="$2" + file="$3" + if [ -n "$value" ]; then + printf '%s=%s\n' "$key" "$value" >> "$file" + fi +} + +show_env_value() { + key="$1" + value="$2" + if [ -z "$value" ]; then + return 0 + fi + if is_secret_key "$key"; then + value=$(mask_secret "$value") + fi + printf ' %s=%s\n' "$key" "$value" +} + +display_env_summary() { + show_env_value COMPOSE_PROJECT_NAME "$COMPOSE_PROJECT_NAME_VALUE" + show_env_value IMAGE_TAG "$IMAGE_TAG_VALUE" + show_env_value WEBHOOK_HOST "$WEBHOOK_HOST_VALUE" + show_env_value MINIAPP_HOST "$MINIAPP_HOST_VALUE" + show_env_value WEBHOOK_PUBLIC_URL "$WEBHOOK_PUBLIC_URL_VALUE" + show_env_value MINIAPP_PUBLIC_URL "$MINIAPP_PUBLIC_URL_VALUE" + show_env_value BOT_TOKEN "$BOT_TOKEN_VALUE" + show_env_value ADMIN_IDS "$ADMIN_IDS_VALUE" + show_env_value POSTGRES_USER "$POSTGRES_USER_VALUE" + show_env_value POSTGRES_PASSWORD "$POSTGRES_PASSWORD_VALUE" + show_env_value POSTGRES_DB "$POSTGRES_DB_VALUE" + show_env_value WEBAPP_SESSION_SECRET "$WEBAPP_SESSION_SECRET_VALUE" + show_env_value WEBHOOK_SECRET_TOKEN "$WEBHOOK_SECRET_TOKEN_VALUE" + show_env_value PANEL_API_URL "$PANEL_API_URL_VALUE" + show_env_value PANEL_API_KEY "$PANEL_API_KEY_VALUE" + show_env_value PANEL_WEBHOOK_SECRET "$PANEL_WEBHOOK_SECRET_VALUE" +} + +append_preserved_env() { + output="$1" + [ -f "$ENV_PATH" ] || return 0 + wrote_header=0 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ""|\#*) + continue + ;; + *=*) + key=${line%%=*} + if known_env_key "$key"; then + continue + fi + if [ "$wrote_header" = "0" ]; then + printf '\n# Preserved from previous .env\n' >> "$output" + wrote_header=1 + fi + printf '%s\n' "$line" >> "$output" + ;; + esac + done < "$ENV_PATH" +} + +render_env_file() { + output="$1" + : > "$output" + printf '# Deployment\n' >> "$output" + env_line COMPOSE_PROJECT_NAME "$COMPOSE_PROJECT_NAME_VALUE" "$output" + env_line IMAGE_TAG "$IMAGE_TAG_VALUE" "$output" + env_line WEBHOOK_HOST "$WEBHOOK_HOST_VALUE" "$output" + env_line MINIAPP_HOST "$MINIAPP_HOST_VALUE" "$output" + env_line WEBHOOK_PUBLIC_URL "$WEBHOOK_PUBLIC_URL_VALUE" "$output" + env_line MINIAPP_PUBLIC_URL "$MINIAPP_PUBLIC_URL_VALUE" "$output" + env_line HTTP_BIND "$HTTP_BIND_VALUE" "$output" + env_line HTTPS_BIND "$HTTPS_BIND_VALUE" "$output" + env_line WEB_SERVER_BIND "$WEB_SERVER_BIND_VALUE" "$output" + env_line FRONTEND_BIND "$FRONTEND_BIND_VALUE" "$output" + env_line PANGOLIN_ENDPOINT "$PANGOLIN_ENDPOINT_VALUE" "$output" + env_line NEWT_ID "$NEWT_ID_VALUE" "$output" + env_line NEWT_SECRET "$NEWT_SECRET_VALUE" "$output" + + printf '\n# Telegram\n' >> "$output" + env_line BOT_TOKEN "$BOT_TOKEN_VALUE" "$output" + env_line ADMIN_IDS "$ADMIN_IDS_VALUE" "$output" + + printf '\n# PostgreSQL\n' >> "$output" + env_line POSTGRES_USER "$POSTGRES_USER_VALUE" "$output" + env_line POSTGRES_PASSWORD "$POSTGRES_PASSWORD_VALUE" "$output" + env_line POSTGRES_DB "$POSTGRES_DB_VALUE" "$output" + + printf '\n# Application\n' >> "$output" + env_line WEBAPP_ENABLED "$WEBAPP_ENABLED_VALUE" "$output" + env_line WEBAPP_SESSION_SECRET "$WEBAPP_SESSION_SECRET_VALUE" "$output" + env_line WEBHOOK_SECRET_TOKEN "$WEBHOOK_SECRET_TOKEN_VALUE" "$output" + env_line TRUSTED_PROXIES "$TRUSTED_PROXIES_VALUE" "$output" + + printf '\n# Remnawave Panel\n' >> "$output" + env_line PANEL_API_URL "$PANEL_API_URL_VALUE" "$output" + env_line PANEL_API_KEY "$PANEL_API_KEY_VALUE" "$output" + env_line PANEL_WEBHOOK_SECRET "$PANEL_WEBHOOK_SECRET_VALUE" "$output" + + append_preserved_env "$output" +} + +write_env_file() { + section "Review .env" + display_env_summary + if ! confirm "Write .env now?" 1; then + warn "Skipped .env write." + return 0 + fi + tmp="$TARGET_DIR/.env.tmp.$$" + render_env_file "$tmp" + if [ -e "$ENV_PATH" ]; then + backup=$(backup_path "$ENV_PATH") + cp "$ENV_PATH" "$backup" + info "Backed up $ENV_PATH to $(basename "$backup")" + fi + mv "$tmp" "$ENV_PATH" + ok "Wrote $ENV_PATH" +} + +prepare_data_directory() { + section "Prepare data directory" + data_dir="$TARGET_DIR/data" + mkdir -p "$data_dir/themes" "$data_dir/webapp-logo" "$data_dir/webapp-emoji" "$data_dir/backups" + if [ ! -f "$data_dir/locales-overrides.json" ]; then + printf '{}\n' > "$data_dir/locales-overrides.json" + fi + if command -v chown >/dev/null 2>&1; then + if ! chown -R "$APP_UID:$APP_GID" "$data_dir" 2>/dev/null; then + warn "Could not chown data files. Run: sudo chown -R $APP_UID:$APP_GID data" + fi + fi + ok "Prepared $data_dir" +} + +require_docker() { + if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + COMPOSE_STYLE="docker" + elif command -v docker-compose >/dev/null 2>&1; then + COMPOSE_STYLE="docker-compose" + else + fail "Docker Compose was not found." + return 1 + fi + if command -v docker >/dev/null 2>&1 && ! docker info >/dev/null 2>&1; then + fail "Docker is installed but not reachable. Check service/user permissions." + return 1 + fi +} + +compose() { + if [ "$COMPOSE_STYLE" = "docker" ]; then + docker compose "$@" + else + docker-compose "$@" + fi +} + +run_compose() { + if [ "$COMPOSE_STYLE" = "docker" ]; then + color "+ docker compose $*" "$DIM" + else + color "+ docker-compose $*" "$DIM" + fi + printf '\n' + compose "$@" +} + +start_stack() { + section "Start Docker stack" + require_docker || return 1 + (cd "$TARGET_DIR" && run_compose pull) || return 1 + (cd "$TARGET_DIR" && run_compose up -d) || return 1 + (cd "$TARGET_DIR" && run_compose ps) || true + ok "Stack command completed." +} + +validate_stack() { + section "Validate stack" + require_docker || return 1 + (cd "$TARGET_DIR" && run_compose ps) || true + (cd "$TARGET_DIR" && run_compose logs --tail 80 migrate) || true + ok "Validation commands completed." +} + +download_importer() { + importer="$TARGET_DIR/$IMPORTER_CACHE_PATH" + mkdir -p "$(dirname "$importer")" + if [ -f "$importer" ] && confirm "Use cached importer at $importer?" 1 >&2; then + printf '%s' "$importer" + return 0 + fi + url=$(raw_url "$SOURCE_REPO" "$SOURCE_REF" "backend/scripts/import_legacy.py") + tmp="$TARGET_DIR/.import_legacy.$$" + download_to "$url" "$tmp" || { + rm -f "$tmp" + fail "Could not download $url" + return 1 + } + if [ -f "$importer" ]; then + backup=$(backup_path "$importer") + cp "$importer" "$backup" + info "Backed up $importer to $(basename "$backup")" >&2 + fi + mv "$tmp" "$importer" + ok "Cached importer at $importer" >&2 + printf '%s' "$importer" +} + +local_target_dsn() { + printf 'postgresql://%s:%s@postgres:5432/%s' "$POSTGRES_USER_VALUE" "$POSTGRES_PASSWORD_VALUE" "$POSTGRES_DB_VALUE" +} + +run_import_command() { + dry="$1" + set -- run --rm \ + -v "$IMPORTER_PATH:/app/backend/scripts/import_legacy.py:ro" + if [ -n "$TARIFF_MAP_PATH" ]; then + set -- "$@" -v "$TARIFF_MAP_PATH:/tmp/tariff-map.json:ro" + fi + set -- "$@" backend python backend/scripts/import_legacy.py \ + --source-type remnashop \ + --source-dsn "$SOURCE_DSN" \ + --source-schema "$SOURCE_SCHEMA" \ + --target-dsn "$TARGET_DSN" + if [ -n "$TARIFF_MAP_PATH" ]; then + set -- "$@" --tariff-map-json /tmp/tariff-map.json + fi + if [ "$dry" = "1" ]; then + set -- "$@" --dry-run + fi + (cd "$TARGET_DIR" && run_compose "$@") +} + +run_legacy_migration() { + section "Legacy migration" + ENV_PATH="$TARGET_DIR/.env" + if [ ! -f "$ENV_PATH" ]; then + fail ".env not found. Install or generate configuration first." + return 1 + fi + require_docker || return 1 + POSTGRES_USER_VALUE="$(env_get POSTGRES_USER '')" + POSTGRES_PASSWORD_VALUE="$(env_get POSTGRES_PASSWORD '')" + POSTGRES_DB_VALUE="$(env_get POSTGRES_DB '')" + + choose "Source bot" "1" "1|2" \ + "1. Remnashop" \ + "2. Skip migration" + [ "$CHOICE_VALUE" = "2" ] && return 0 + + prompt_value "Source Remnashop PostgreSQL DSN" "${REMNASHOP_SOURCE_DSN:-}" 1 0 "" + SOURCE_DSN="$PROMPT_VALUE" + prompt_value "Source schema" "public" 1 0 "" + SOURCE_SCHEMA="$PROMPT_VALUE" + + choose "Target database" "1" "1|2" \ + "1. This Docker Compose stack database (recommended)" \ + "2. Manual target DSN" + if [ "$CHOICE_VALUE" = "1" ]; then + TARGET_DSN="$(local_target_dsn)" + info "Target DSN points to the Compose postgres service." + else + prompt_value "Target PostgreSQL DSN" "" 1 0 "" + TARGET_DSN="$PROMPT_VALUE" + fi + + prompt_value "Optional tariff map JSON path (empty to skip)" "" 0 0 "" + TARIFF_MAP_PATH="$PROMPT_VALUE" + if [ -n "$TARIFF_MAP_PATH" ]; then + tariff_map_dir=$(dirname "$TARIFF_MAP_PATH") + if [ ! -d "$tariff_map_dir" ]; then + fail "Tariff map directory not found: $tariff_map_dir" + return 1 + fi + TARIFF_MAP_PATH=$(cd "$tariff_map_dir" && pwd)/$(basename "$TARIFF_MAP_PATH") + if [ ! -f "$TARIFF_MAP_PATH" ]; then + fail "Tariff map not found: $TARIFF_MAP_PATH" + return 1 + fi + fi + + IMPORTER_PATH="$(download_importer)" || return 1 + + section "Dry-run import" + if ! run_import_command 1; then + fail "Dry-run failed. Fix the connection/settings before importing." + return 1 + fi + if ! confirm "Apply this migration for real?" 0; then + warn "Migration not applied." + return 0 + fi + + section "Apply import" + run_import_command 0 || return 1 + if confirm "Restart backend and worker so setting overrides are reloaded?" 1; then + (cd "$TARGET_DIR" && run_compose restart backend worker) || true + fi + ok "Legacy migration completed." +} + +installation_directory() { + prompt_value "Install directory" "${MINISHOP_INSTALL_DIR:-$(pwd)}" 1 0 "" + mkdir -p "$(dirname "$PROMPT_VALUE")" + TARGET_DIR=$(cd "$(dirname "$PROMPT_VALUE")" && pwd)/$(basename "$PROMPT_VALUE") + mkdir -p "$TARGET_DIR" +} + +github_source() { + prompt_value "GitHub repository" "$DEFAULT_REPO" 1 0 "" + SOURCE_REPO="$PROMPT_VALUE" + prompt_value "Git ref/branch/tag for raw files" "$DEFAULT_REF" 1 0 "" + SOURCE_REF="$PROMPT_VALUE" +} + +install_flow() { + with_migration="$1" + installation_directory || return 1 + github_source || return 1 + choose_profile + ENV_PATH="$TARGET_DIR/.env" + if [ -f "$ENV_PATH" ]; then + warn "Existing .env found at $ENV_PATH; wizard will preserve unknown values." + fi + prompt_common_env || return 1 + download_profile_files || return 1 + write_env_file || return 1 + mkdir -p "$TARGET_DIR/$INSTALL_STATE_DIR" + prepare_data_directory || return 1 + if confirm "Start Docker Compose stack now?" 1; then + start_stack || return 1 + fi + if [ "$with_migration" = "1" ]; then + run_legacy_migration + elif confirm "Run a legacy bot migration now?" 0; then + run_legacy_migration + fi +} + +migration_only_flow() { + installation_directory || return 1 + github_source || return 1 + run_legacy_migration +} + +download_only_flow() { + installation_directory || return 1 + github_source || return 1 + choose_profile + download_profile_files +} + +health_flow() { + installation_directory || return 1 + validate_stack +} + +main_menu() { + while :; do + banner + choose "Main menu" "1" "1|2|3|4|5|6" \ + "1. Install new stack" \ + "2. Install new stack and run legacy migration" \ + "3. Run legacy migration only" \ + "4. Download/update deployment files only" \ + "5. Validate current stack" \ + "6. Exit" + case "$CHOICE_VALUE" in + 1) install_flow 0 ;; + 2) install_flow 1 ;; + 3) migration_only_flow ;; + 4) download_only_flow ;; + 5) health_flow ;; + 6) printf 'Bye.\n'; return 0 ;; + esac + status=$? + if [ "$status" -ne 0 ]; then + fail "Step failed with status $status." + fi + pause + done +} + +case "${1:-}" in + -h|--help) + print_help + exit 0 + ;; +esac + +main_menu diff --git a/tests/test_install_script.py b/tests/test_install_script.py new file mode 100644 index 0000000..b185524 --- /dev/null +++ b/tests/test_install_script.py @@ -0,0 +1,39 @@ +import shutil +import subprocess +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +INSTALL_SCRIPT = REPO_ROOT / "scripts" / "install.sh" + + +def test_shell_installer_help_does_not_require_python(): + if not shutil.which("sh"): + pytest.skip("sh is not available on this platform") + + result = subprocess.run( + ["sh", str(INSTALL_SCRIPT), "--help"], + check=True, + text=True, + capture_output=True, + ) + + assert "MINISHOP_INSTALL_REPO" in result.stdout + assert "dry-run" in result.stdout + + +def test_shell_installer_is_the_only_install_entrypoint(): + assert INSTALL_SCRIPT.exists() + assert not (REPO_ROOT / "scripts" / "install.py").exists() + + +def test_shell_installer_downloads_raw_files_and_runs_import_in_container(): + script = INSTALL_SCRIPT.read_text(encoding="utf-8") + + assert script.startswith("#!/bin/sh") + assert "raw.githubusercontent.com" in script + assert "git clone" not in script + assert "backend python backend/scripts/import_legacy.py" in script + assert "--dry-run" in script + assert "Install new stack and run legacy migration" in script From 3b40369606f91735c43e56653a6b596917afbc4c Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 10:17:50 +0300 Subject: [PATCH 03/11] feat: add legacy tg-shop migration to installer --- docs/getting-started/deployment.md | 10 +- docs/migrations/remnawave-tg-shop.md | 27 +++ scripts/install.sh | 285 +++++++++++++++++++++++++-- tests/test_install_script.py | 11 ++ 4 files changed, 314 insertions(+), 19 deletions(-) diff --git a/docs/getting-started/deployment.md b/docs/getting-started/deployment.md index ae3f09d..844cdda 100644 --- a/docs/getting-started/deployment.md +++ b/docs/getting-started/deployment.md @@ -31,7 +31,8 @@ Wizard работает через меню с цифрами и подтвер - сохранить backup существующих файлов перед перезаписью; - запустить `docker compose pull && docker compose up -d`; - проверить текущий стек через `docker compose ps` и логи `migrate`; -- запустить миграцию из поддерживаемых legacy-ботов, сейчас - из Remnashop. +- запустить миграцию из поддерживаемых legacy-ботов: Remnashop и старый + `remnawave-tg-shop`; Для тестирования другой ветки или форка задайте источник перед запуском: @@ -41,8 +42,11 @@ MINISHOP_INSTALL_REF=main \ sh install.sh ``` -Миграция в wizard сначала запускает `dry-run`, показывает JSON-сводку и только -после отдельного подтверждения применяет изменения в целевую БД. +Миграция Remnashop в wizard сначала запускает `dry-run`, показывает JSON-сводку +и только после отдельного подтверждения применяет изменения в целевую БД. +Миграция со старого `remnawave-tg-shop` работает как upgrade совместимой БД: +либо копирует старый Docker volume, либо делает `pg_dump` по source DSN, +восстанавливает дамп в целевую compose-БД и запускает сервис `migrate`. Обычный `docker compose up -d --build` поднимает: diff --git a/docs/migrations/remnawave-tg-shop.md b/docs/migrations/remnawave-tg-shop.md index b7e16b0..9ef4b84 100644 --- a/docs/migrations/remnawave-tg-shop.md +++ b/docs/migrations/remnawave-tg-shop.md @@ -4,6 +4,33 @@ Автоматический скрипт ниже рассчитан именно на родственный стек `remnawave-tg-shop`, где структура БД и Docker volumes известны заранее. Для других ботов нужен отдельный адаптер экспорта/импорта. +## Новый install wizard + +Для нового сервера или переноса без клонирования репозитория используйте общий +`sh` wizard: + +```bash +curl -fsSL https://raw.githubusercontent.com/3252a8/remnawave-minishop/main/scripts/install.sh -o install.sh +sh install.sh +``` + +В меню выберите `Install new stack and run legacy migration` или +`Run legacy migration only`, затем источник `Legacy remnawave-tg-shop`. +Wizard поддерживает два режима: + +- `Copy old Docker volumes` - тот же безопасный сценарий, что и старый + `migrate_to_minishop.sh`: старый volume `remnawave-tg-shop-db-data` + копируется в `remnawave-minishop-db-data`, затем новый stack запускает + сервис `migrate` и накатывает все схемные миграции; +- `Dump from a source PostgreSQL DSN` - новый режим для случаев, когда старая + БД доступна как внешний PostgreSQL DSN. Wizard поднимает целевой `postgres`, + сбрасывает целевую БД, делает `pg_dump` из старой БД, восстанавливает дамп в + compose-БД и затем запускает `migrate`. + +Старый helper `scripts/migrate_to_minishop.sh` ниже всё ещё полезен для +in-place обновления уже клонированного репозитория: он переключает git-ветку, +переносит volumes и стартует новый stack. Прямой source DSN он не поддерживал. + ## Короткий путь без смены ветки и сборки Если вы используете только готовые Docker-образы и не собираете проект diff --git a/scripts/install.sh b/scripts/install.sh index d9d93dc..f365310 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -11,6 +11,13 @@ INSTALL_STATE_DIR=".installer" IMPORTER_CACHE_PATH="$INSTALL_STATE_DIR/import_legacy.py" APP_UID=10001 APP_GID=10001 +OLD_TGSHOP_DB_VOLUME="remnawave-tg-shop-db-data" +NEW_MINISHOP_DB_VOLUME="remnawave-minishop-db-data" +OLD_TGSHOP_CADDY_DATA_VOLUME="remnawave-tg-shop-caddy-data" +OLD_TGSHOP_CADDY_CONFIG_VOLUME="remnawave-tg-shop-caddy-config" +NEW_MINISHOP_CADDY_DATA_VOLUME="remnawave-minishop-caddy-data" +NEW_MINISHOP_CADDY_CONFIG_VOLUME="remnawave-minishop-caddy-config" +KNOWN_LEGACY_CONTAINERS="remnawave-tg-shop remnawave-tg-shop-db remnawave-tg-shop-caddy remnawave-minishop remnawave-minishop-db remnawave-minishop-caddy remnawave-minishop-backend remnawave-minishop-worker remnawave-minishop-frontend remnawave-minishop-migrate remnawave-minishop-postgres remnawave-minishop-redis" if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then RESET="$(printf '\033[0m')" @@ -40,6 +47,7 @@ ENV_PATH="" COMPOSE_STYLE="" PROMPT_VALUE="" CHOICE_VALUE="" +LEGACY_SOURCE="" COMPOSE_PROJECT_NAME_VALUE="" IMAGE_TAG_VALUE="" @@ -121,9 +129,11 @@ Environment overrides: MINISHOP_INSTALL_REF default ref ($DEFAULT_REF) MINISHOP_IMAGE_TAG default image tag ($DEFAULT_IMAGE_TAG) REMNASHOP_SOURCE_DSN default source DSN for migration + LEGACY_TGSHOP_SOURCE_DSN default remnawave-tg-shop source DSN for dump/restore The wizard is interactive by design. It never overwrites files without -confirmation and always runs legacy imports as dry-run first. +confirmation. Remnashop imports always run dry-run first; legacy +remnawave-tg-shop can be migrated from Docker volumes or a PostgreSQL DSN. EOF } @@ -677,9 +687,12 @@ run_compose() { } start_stack() { + pull="${1:-1}" section "Start Docker stack" require_docker || return 1 - (cd "$TARGET_DIR" && run_compose pull) || return 1 + if [ "$pull" = "1" ]; then + (cd "$TARGET_DIR" && run_compose pull) || return 1 + fi (cd "$TARGET_DIR" && run_compose up -d) || return 1 (cd "$TARGET_DIR" && run_compose ps) || true ok "Stack command completed." @@ -693,6 +706,95 @@ validate_stack() { ok "Validation commands completed." } +volume_exists() { + docker volume inspect "$1" >/dev/null 2>&1 +} + +volume_is_empty() { + docker run --rm -v "$1:/data" alpine sh -c \ + 'test -z "$(find /data -mindepth 1 -print -quit)"' >/dev/null 2>&1 +} + +copy_volume_if_safe() { + source_volume="$1" + target_volume="$2" + required="${3:-0}" + + if ! volume_exists "$source_volume"; then + if [ "$required" = "1" ]; then + fail "Source Docker volume not found: $source_volume" + return 1 + fi + warn "Skipping $source_volume: source volume was not found." + return 0 + fi + + if ! volume_exists "$target_volume"; then + if [ "$required" = "1" ]; then + fail "Target Docker volume not found: $target_volume" + return 1 + fi + warn "Skipping $target_volume: target volume was not created by this profile." + return 0 + fi + + if ! volume_is_empty "$target_volume"; then + if [ "$required" = "1" ]; then + warn "Target volume $target_volume is already not empty." + warn "It may already be migrated, or the target stack may have been started with an empty database." + if confirm "Continue without copying the legacy database volume?" 0; then + return 0 + fi + return 1 + fi + warn "Skipping $target_volume: target volume is already not empty." + return 0 + fi + + run_label="docker run --rm -v $source_volume:/from:ro -v $target_volume:/to alpine sh -c 'cd /from && cp -a . /to/'" + color "+ $run_label" "$DIM" + printf '\n' + docker run --rm \ + -v "$source_volume:/from:ro" \ + -v "$target_volume:/to" \ + alpine sh -c 'cd /from && cp -a . /to/' || return 1 + ok "Copied $source_volume -> $target_volume" +} + +stop_known_legacy_containers() { + section "Stop legacy containers" + stopped=0 + for container in $KNOWN_LEGACY_CONTAINERS; do + if docker inspect "$container" >/dev/null 2>&1; then + if docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null | grep -q '^true$'; then + docker stop "$container" >/dev/null || true + fi + docker rm "$container" >/dev/null || true + info "Stopped/removed $container" + stopped=1 + fi + done + if [ "$stopped" = "0" ]; then + info "No known legacy containers found." + fi +} + +wait_target_postgres() { + section "Wait for target PostgreSQL" + attempt=1 + while [ "$attempt" -le 30 ]; do + if (cd "$TARGET_DIR" && compose exec -T postgres sh -c \ + 'pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"' >/dev/null 2>&1); then + ok "PostgreSQL is ready." + return 0 + fi + sleep 2 + attempt=$((attempt + 1)) + done + fail "Target PostgreSQL did not become ready." + return 1 +} + download_importer() { importer="$TARGET_DIR/$IMPORTER_CACHE_PATH" mkdir -p "$(dirname "$importer")" @@ -742,23 +844,38 @@ run_import_command() { (cd "$TARGET_DIR" && run_compose "$@") } -run_legacy_migration() { - section "Legacy migration" +choose_legacy_source() { + choose "Source bot" "1" "1|2|3" \ + "1. Remnashop - import users, subscriptions, payments, referrals and promo codes." \ + "2. Legacy remnawave-tg-shop - upgrade an old compatible database/volume." \ + "3. Skip migration" + case "$CHOICE_VALUE" in + 1) LEGACY_SOURCE="remnashop" ;; + 2) LEGACY_SOURCE="remnawave-tg-shop" ;; + 3) LEGACY_SOURCE="skip" ;; + esac +} + +ensure_github_source_for_importer() { + if [ -n "$SOURCE_REPO" ] && [ -n "$SOURCE_REF" ]; then + return 0 + fi + github_source +} + +run_remnashop_migration() { + section "Remnashop migration" ENV_PATH="$TARGET_DIR/.env" if [ ! -f "$ENV_PATH" ]; then fail ".env not found. Install or generate configuration first." return 1 fi + ensure_github_source_for_importer || return 1 require_docker || return 1 POSTGRES_USER_VALUE="$(env_get POSTGRES_USER '')" POSTGRES_PASSWORD_VALUE="$(env_get POSTGRES_PASSWORD '')" POSTGRES_DB_VALUE="$(env_get POSTGRES_DB '')" - choose "Source bot" "1" "1|2" \ - "1. Remnashop" \ - "2. Skip migration" - [ "$CHOICE_VALUE" = "2" ] && return 0 - prompt_value "Source Remnashop PostgreSQL DSN" "${REMNASHOP_SOURCE_DSN:-}" 1 0 "" SOURCE_DSN="$PROMPT_VALUE" prompt_value "Source schema" "public" 1 0 "" @@ -810,6 +927,118 @@ run_legacy_migration() { ok "Legacy migration completed." } +run_target_schema_migrations() { + section "Apply target schema migrations" + require_docker || return 1 + (cd "$TARGET_DIR" && run_compose run --rm migrate) || return 1 + ok "Schema migrations completed." +} + +prepare_compose_without_starting_apps() { + section "Prepare target Compose stack" + require_docker || return 1 + (cd "$TARGET_DIR" && run_compose up --no-start) || return 1 +} + +run_tgshop_volume_migration() { + section "Legacy remnawave-tg-shop volume migration" + warn "This path copies the old PostgreSQL Docker volume into the new Minishop volume." + warn "Old volumes are not deleted; keep them until you verify the new stack." + + if confirm "Stop known old/current containers before copying volumes?" 1; then + stop_known_legacy_containers || return 1 + (cd "$TARGET_DIR" && run_compose down) || true + fi + + prepare_compose_without_starting_apps || return 1 + copy_volume_if_safe "$OLD_TGSHOP_DB_VOLUME" "$NEW_MINISHOP_DB_VOLUME" 1 || return 1 + copy_volume_if_safe "$OLD_TGSHOP_CADDY_DATA_VOLUME" "$NEW_MINISHOP_CADDY_DATA_VOLUME" 0 || return 1 + copy_volume_if_safe "$OLD_TGSHOP_CADDY_CONFIG_VOLUME" "$NEW_MINISHOP_CADDY_CONFIG_VOLUME" 0 || return 1 + + if confirm "Start the new stack and let migrate apply schema changes now?" 1; then + start_stack 0 || return 1 + (cd "$TARGET_DIR" && run_compose logs --tail 120 migrate) || true + else + warn "Stack was prepared but not started. Run docker compose up -d later." + fi +} + +run_tgshop_dsn_migration() { + section "Legacy remnawave-tg-shop DSN migration" + warn "The old standalone helper did not support direct DSN import." + warn "This wizard path dumps the old PostgreSQL database, restores it into target Compose PostgreSQL, then runs Minishop schema migrations." + warn "The target database will be dropped and recreated before restore." + + if ! confirm "Replace target database with the legacy dump?" 0; then + warn "Migration not applied." + return 0 + fi + + prompt_value "Source remnawave-tg-shop PostgreSQL DSN" "${LEGACY_TGSHOP_SOURCE_DSN:-}" 1 0 "" + SOURCE_DSN="$PROMPT_VALUE" + + require_docker || return 1 + POSTGRES_USER_VALUE="$(env_get POSTGRES_USER '')" + POSTGRES_PASSWORD_VALUE="$(env_get POSTGRES_PASSWORD '')" + POSTGRES_DB_VALUE="$(env_get POSTGRES_DB '')" + TARGET_DSN="$(local_target_dsn)" + + section "Start target PostgreSQL" + (cd "$TARGET_DIR" && run_compose stop backend worker frontend migrate) || true + (cd "$TARGET_DIR" && run_compose up -d postgres redis) || return 1 + wait_target_postgres || return 1 + + section "Reset target database" + (cd "$TARGET_DIR" && run_compose exec -T postgres sh -c \ + 'dropdb -U "$POSTGRES_USER" --if-exists "$POSTGRES_DB" && createdb -U "$POSTGRES_USER" "$POSTGRES_DB"') || return 1 + + section "Dump and restore legacy database" + (cd "$TARGET_DIR" && run_compose run --rm --no-deps \ + -e "SOURCE_DSN=$SOURCE_DSN" \ + -e "TARGET_DSN=$TARGET_DSN" \ + backend sh -lc \ + 'pg_dump --clean --if-exists --no-owner --no-privileges "$SOURCE_DSN" | psql "$TARGET_DSN"') || return 1 + + run_target_schema_migrations || return 1 + if confirm "Start the full stack now?" 1; then + start_stack 0 || return 1 + fi +} + +run_remnawave_tg_shop_migration() { + section "Legacy remnawave-tg-shop migration" + ENV_PATH="$TARGET_DIR/.env" + if [ ! -f "$ENV_PATH" ]; then + fail ".env not found. Install or generate configuration first." + return 1 + fi + require_docker || return 1 + + choose "Migration method" "1" "1|2|3" \ + "1. Copy old Docker volumes on this host (recommended for old compose installs)." \ + "2. Dump from a source PostgreSQL DSN and restore into this compose stack." \ + "3. Skip migration" + case "$CHOICE_VALUE" in + 1) run_tgshop_volume_migration ;; + 2) run_tgshop_dsn_migration ;; + 3) return 0 ;; + esac +} + +run_selected_legacy_migration() { + case "$LEGACY_SOURCE" in + remnashop) + run_remnashop_migration + ;; + remnawave-tg-shop) + run_remnawave_tg_shop_migration + ;; + skip|"") + return 0 + ;; + esac +} + installation_directory() { prompt_value "Install directory" "${MINISHOP_INSTALL_DIR:-$(pwd)}" 1 0 "" mkdir -p "$(dirname "$PROMPT_VALUE")" @@ -826,6 +1055,7 @@ github_source() { install_flow() { with_migration="$1" + LEGACY_SOURCE="" installation_directory || return 1 github_source || return 1 choose_profile @@ -838,20 +1068,43 @@ install_flow() { write_env_file || return 1 mkdir -p "$TARGET_DIR/$INSTALL_STATE_DIR" prepare_data_directory || return 1 - if confirm "Start Docker Compose stack now?" 1; then - start_stack || return 1 - fi if [ "$with_migration" = "1" ]; then - run_legacy_migration + choose_legacy_source elif confirm "Run a legacy bot migration now?" 0; then - run_legacy_migration + choose_legacy_source fi + + case "$LEGACY_SOURCE" in + remnawave-tg-shop) + run_selected_legacy_migration + ;; + remnashop) + if confirm "Start Docker Compose stack before Remnashop import?" 1; then + start_stack || return 1 + else + warn "Remnashop import needs the target stack database. Skipping import." + return 0 + fi + run_selected_legacy_migration + ;; + *) + if confirm "Start Docker Compose stack now?" 1; then + start_stack || return 1 + fi + ;; + esac } migration_only_flow() { + LEGACY_SOURCE="" installation_directory || return 1 - github_source || return 1 - run_legacy_migration + choose_legacy_source + case "$LEGACY_SOURCE" in + remnashop) + github_source || return 1 + ;; + esac + run_selected_legacy_migration } download_only_flow() { diff --git a/tests/test_install_script.py b/tests/test_install_script.py index b185524..cec0b1e 100644 --- a/tests/test_install_script.py +++ b/tests/test_install_script.py @@ -21,6 +21,7 @@ def test_shell_installer_help_does_not_require_python(): assert "MINISHOP_INSTALL_REPO" in result.stdout assert "dry-run" in result.stdout + assert "LEGACY_TGSHOP_SOURCE_DSN" in result.stdout def test_shell_installer_is_the_only_install_entrypoint(): @@ -37,3 +38,13 @@ def test_shell_installer_downloads_raw_files_and_runs_import_in_container(): assert "backend python backend/scripts/import_legacy.py" in script assert "--dry-run" in script assert "Install new stack and run legacy migration" in script + + +def test_shell_installer_supports_legacy_tgshop_volume_and_dsn_paths(): + script = INSTALL_SCRIPT.read_text(encoding="utf-8") + + assert "Legacy remnawave-tg-shop" in script + assert "remnawave-tg-shop-db-data" in script + assert "remnawave-minishop-db-data" in script + assert "pg_dump --clean --if-exists" in script + assert "run_compose run --rm migrate" in script From 162e9e20b591d5d7d38340dc3b493db7ca644559 Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 10:21:08 +0300 Subject: [PATCH 04/11] fix: keep installer from preparing data directories --- docs/getting-started/deployment.md | 6 ++-- docs/migrations/remnawave-tg-shop.md | 2 +- scripts/install.sh | 45 ++++++++-------------------- tests/test_install_script.py | 2 +- 4 files changed, 17 insertions(+), 38 deletions(-) diff --git a/docs/getting-started/deployment.md b/docs/getting-started/deployment.md index 844cdda..d3d04cc 100644 --- a/docs/getting-started/deployment.md +++ b/docs/getting-started/deployment.md @@ -15,9 +15,7 @@ docker compose logs -f backend worker frontend ## Интерактивный install wizard -Для нового сервера можно не клонировать весь репозиторий и не ставить Python. -Скачайте только POSIX `sh`-скрипт через `raw.githubusercontent.com` и -запустите его: +Для нового сервера скачайте install-скрипт и запустите его: ```bash curl -fsSL https://raw.githubusercontent.com/3252a8/remnawave-minishop/main/scripts/install.sh -o install.sh @@ -31,7 +29,7 @@ Wizard работает через меню с цифрами и подтвер - сохранить backup существующих файлов перед перезаписью; - запустить `docker compose pull && docker compose up -d`; - проверить текущий стек через `docker compose ps` и логи `migrate`; -- запустить миграцию из поддерживаемых legacy-ботов: Remnashop и старый +- запустить миграцию из поддерживаемых ботов: Remnashop и старый `remnawave-tg-shop`; Для тестирования другой ветки или форка задайте источник перед запуском: diff --git a/docs/migrations/remnawave-tg-shop.md b/docs/migrations/remnawave-tg-shop.md index 9ef4b84..789fa71 100644 --- a/docs/migrations/remnawave-tg-shop.md +++ b/docs/migrations/remnawave-tg-shop.md @@ -15,7 +15,7 @@ sh install.sh ``` В меню выберите `Install new stack and run legacy migration` или -`Run legacy migration only`, затем источник `Legacy remnawave-tg-shop`. +`Run legacy migration only`, затем источник `Old remnawave-tg-shop`. Wizard поддерживает два режима: - `Copy old Docker volumes` - тот же безопасный сценарий, что и старый diff --git a/scripts/install.sh b/scripts/install.sh index f365310..bc7c57c 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1,16 +1,13 @@ #!/bin/sh set -u -# Dependency-light installer for fresh Linux hosts. -# It intentionally avoids Python and clones only the files selected by the user. +# Interactive installer for fresh Docker Compose hosts. DEFAULT_REPO="${MINISHOP_INSTALL_REPO:-3252a8/remnawave-minishop}" DEFAULT_REF="${MINISHOP_INSTALL_REF:-main}" DEFAULT_IMAGE_TAG="${MINISHOP_IMAGE_TAG:-latest}" INSTALL_STATE_DIR=".installer" IMPORTER_CACHE_PATH="$INSTALL_STATE_DIR/import_legacy.py" -APP_UID=10001 -APP_GID=10001 OLD_TGSHOP_DB_VOLUME="remnawave-tg-shop-db-data" NEW_MINISHOP_DB_VOLUME="remnawave-minishop-db-data" OLD_TGSHOP_CADDY_DATA_VOLUME="remnawave-tg-shop-caddy-data" @@ -85,7 +82,7 @@ banner() { printf '\n' color "Remnawave MiniShop Install Wizard" "$BOLD$CYAN" printf '\n' - color "Install, configure, start, and migrate legacy bot data." "$DIM" + color "Install, configure, start, and migrate existing bot data." "$DIM" printf '\n\n' } @@ -132,8 +129,8 @@ Environment overrides: LEGACY_TGSHOP_SOURCE_DSN default remnawave-tg-shop source DSN for dump/restore The wizard is interactive by design. It never overwrites files without -confirmation. Remnashop imports always run dry-run first; legacy -remnawave-tg-shop can be migrated from Docker volumes or a PostgreSQL DSN. +confirmation. Remnashop imports always run dry-run first; remnawave-tg-shop +can be migrated from Docker volumes or a PostgreSQL DSN. EOF } @@ -638,21 +635,6 @@ write_env_file() { ok "Wrote $ENV_PATH" } -prepare_data_directory() { - section "Prepare data directory" - data_dir="$TARGET_DIR/data" - mkdir -p "$data_dir/themes" "$data_dir/webapp-logo" "$data_dir/webapp-emoji" "$data_dir/backups" - if [ ! -f "$data_dir/locales-overrides.json" ]; then - printf '{}\n' > "$data_dir/locales-overrides.json" - fi - if command -v chown >/dev/null 2>&1; then - if ! chown -R "$APP_UID:$APP_GID" "$data_dir" 2>/dev/null; then - warn "Could not chown data files. Run: sudo chown -R $APP_UID:$APP_GID data" - fi - fi - ok "Prepared $data_dir" -} - require_docker() { if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then COMPOSE_STYLE="docker" @@ -742,7 +724,7 @@ copy_volume_if_safe() { if [ "$required" = "1" ]; then warn "Target volume $target_volume is already not empty." warn "It may already be migrated, or the target stack may have been started with an empty database." - if confirm "Continue without copying the legacy database volume?" 0; then + if confirm "Continue without copying the old database volume?" 0; then return 0 fi return 1 @@ -762,7 +744,7 @@ copy_volume_if_safe() { } stop_known_legacy_containers() { - section "Stop legacy containers" + section "Stop old containers" stopped=0 for container in $KNOWN_LEGACY_CONTAINERS; do if docker inspect "$container" >/dev/null 2>&1; then @@ -775,7 +757,7 @@ stop_known_legacy_containers() { fi done if [ "$stopped" = "0" ]; then - info "No known legacy containers found." + info "No known old containers found." fi } @@ -847,7 +829,7 @@ run_import_command() { choose_legacy_source() { choose "Source bot" "1" "1|2|3" \ "1. Remnashop - import users, subscriptions, payments, referrals and promo codes." \ - "2. Legacy remnawave-tg-shop - upgrade an old compatible database/volume." \ + "2. Old remnawave-tg-shop - upgrade an old compatible database/volume." \ "3. Skip migration" case "$CHOICE_VALUE" in 1) LEGACY_SOURCE="remnashop" ;; @@ -941,7 +923,7 @@ prepare_compose_without_starting_apps() { } run_tgshop_volume_migration() { - section "Legacy remnawave-tg-shop volume migration" + section "Old remnawave-tg-shop volume migration" warn "This path copies the old PostgreSQL Docker volume into the new Minishop volume." warn "Old volumes are not deleted; keep them until you verify the new stack." @@ -964,7 +946,7 @@ run_tgshop_volume_migration() { } run_tgshop_dsn_migration() { - section "Legacy remnawave-tg-shop DSN migration" + section "Old remnawave-tg-shop DSN migration" warn "The old standalone helper did not support direct DSN import." warn "This wizard path dumps the old PostgreSQL database, restores it into target Compose PostgreSQL, then runs Minishop schema migrations." warn "The target database will be dropped and recreated before restore." @@ -992,7 +974,7 @@ run_tgshop_dsn_migration() { (cd "$TARGET_DIR" && run_compose exec -T postgres sh -c \ 'dropdb -U "$POSTGRES_USER" --if-exists "$POSTGRES_DB" && createdb -U "$POSTGRES_USER" "$POSTGRES_DB"') || return 1 - section "Dump and restore legacy database" + section "Dump and restore old database" (cd "$TARGET_DIR" && run_compose run --rm --no-deps \ -e "SOURCE_DSN=$SOURCE_DSN" \ -e "TARGET_DSN=$TARGET_DSN" \ @@ -1006,7 +988,7 @@ run_tgshop_dsn_migration() { } run_remnawave_tg_shop_migration() { - section "Legacy remnawave-tg-shop migration" + section "Old remnawave-tg-shop migration" ENV_PATH="$TARGET_DIR/.env" if [ ! -f "$ENV_PATH" ]; then fail ".env not found. Install or generate configuration first." @@ -1067,10 +1049,9 @@ install_flow() { download_profile_files || return 1 write_env_file || return 1 mkdir -p "$TARGET_DIR/$INSTALL_STATE_DIR" - prepare_data_directory || return 1 if [ "$with_migration" = "1" ]; then choose_legacy_source - elif confirm "Run a legacy bot migration now?" 0; then + elif confirm "Run a migration from another bot now?" 0; then choose_legacy_source fi diff --git a/tests/test_install_script.py b/tests/test_install_script.py index cec0b1e..f69a83b 100644 --- a/tests/test_install_script.py +++ b/tests/test_install_script.py @@ -43,7 +43,7 @@ def test_shell_installer_downloads_raw_files_and_runs_import_in_container(): def test_shell_installer_supports_legacy_tgshop_volume_and_dsn_paths(): script = INSTALL_SCRIPT.read_text(encoding="utf-8") - assert "Legacy remnawave-tg-shop" in script + assert "Old remnawave-tg-shop" in script assert "remnawave-tg-shop-db-data" in script assert "remnawave-minishop-db-data" in script assert "pg_dump --clean --if-exists" in script From 0473829f7a64152373f6583a0e8e678c8dec2769 Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 10:30:43 +0300 Subject: [PATCH 05/11] fix: prepare writable data mount in installer --- docs/getting-started/deployment.md | 1 + scripts/install.sh | 36 ++++++++++++++++++++++++++++++ tests/test_install_script.py | 12 ++++++++++ 3 files changed, 49 insertions(+) diff --git a/docs/getting-started/deployment.md b/docs/getting-started/deployment.md index d3d04cc..1df27b0 100644 --- a/docs/getting-started/deployment.md +++ b/docs/getting-started/deployment.md @@ -27,6 +27,7 @@ Wizard работает через меню с цифрами и подтвер - скачать выбранный compose-профиль (`Caddy`, `Nginx`, `Pangolin/Newt` или `no-proxy`); - сгенерировать минимальный `.env`, включая пароли и стабильные secrets; - сохранить backup существующих файлов перед перезаписью; +- подготовить writable `data/` для файлов приложения; - запустить `docker compose pull && docker compose up -d`; - проверить текущий стек через `docker compose ps` и логи `migrate`; - запустить миграцию из поддерживаемых ботов: Remnashop и старый diff --git a/scripts/install.sh b/scripts/install.sh index bc7c57c..26274be 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -8,6 +8,8 @@ DEFAULT_REF="${MINISHOP_INSTALL_REF:-main}" DEFAULT_IMAGE_TAG="${MINISHOP_IMAGE_TAG:-latest}" INSTALL_STATE_DIR=".installer" IMPORTER_CACHE_PATH="$INSTALL_STATE_DIR/import_legacy.py" +APP_UID=10001 +APP_GID=10001 OLD_TGSHOP_DB_VOLUME="remnawave-tg-shop-db-data" NEW_MINISHOP_DB_VOLUME="remnawave-minishop-db-data" OLD_TGSHOP_CADDY_DATA_VOLUME="remnawave-tg-shop-caddy-data" @@ -635,6 +637,37 @@ write_env_file() { ok "Wrote $ENV_PATH" } +prepare_data_mount() { + section "Prepare data mount" + data_dir="$TARGET_DIR/data" + created=0 + if [ ! -d "$data_dir" ]; then + mkdir -p "$data_dir" || return 1 + created=1 + fi + + if [ "$created" = "1" ]; then + if command -v chown >/dev/null 2>&1; then + if ! chown "$APP_UID:$APP_GID" "$data_dir" 2>/dev/null; then + warn "Could not chown $data_dir. Run: sudo chown $APP_UID:$APP_GID data" + fi + fi + chmod u+rwx "$data_dir" 2>/dev/null || true + ok "Created writable $data_dir" + return 0 + fi + + info "$data_dir already exists." + if confirm "Adjust $data_dir owner to $APP_UID:$APP_GID for container writes?" 0; then + if command -v chown >/dev/null 2>&1; then + if ! chown "$APP_UID:$APP_GID" "$data_dir" 2>/dev/null; then + warn "Could not chown $data_dir. Run: sudo chown $APP_UID:$APP_GID data" + fi + fi + chmod u+rwx "$data_dir" 2>/dev/null || true + fi +} + require_docker() { if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then COMPOSE_STYLE="docker" @@ -1049,6 +1082,7 @@ install_flow() { download_profile_files || return 1 write_env_file || return 1 mkdir -p "$TARGET_DIR/$INSTALL_STATE_DIR" + prepare_data_mount || return 1 if [ "$with_migration" = "1" ]; then choose_legacy_source elif confirm "Run a migration from another bot now?" 0; then @@ -1080,6 +1114,8 @@ migration_only_flow() { LEGACY_SOURCE="" installation_directory || return 1 choose_legacy_source + [ "$LEGACY_SOURCE" = "skip" ] && return 0 + prepare_data_mount || return 1 case "$LEGACY_SOURCE" in remnashop) github_source || return 1 diff --git a/tests/test_install_script.py b/tests/test_install_script.py index f69a83b..72a3b56 100644 --- a/tests/test_install_script.py +++ b/tests/test_install_script.py @@ -48,3 +48,15 @@ def test_shell_installer_supports_legacy_tgshop_volume_and_dsn_paths(): assert "remnawave-minishop-db-data" in script assert "pg_dump --clean --if-exists" in script assert "run_compose run --rm migrate" in script + + +def test_shell_installer_only_prepares_data_mount_not_runtime_content(): + script = INSTALL_SCRIPT.read_text(encoding="utf-8") + + assert 'data_dir="$TARGET_DIR/data"' in script + assert "mkdir -p \"$data_dir\"" in script + assert "chown \"$APP_UID:$APP_GID\" \"$data_dir\"" in script + assert "data_dir/themes" not in script + assert "webapp-logo" not in script + assert "webapp-emoji" not in script + assert "locales-overrides.json" not in script From 2ccead9b49b40aa236f19dc7d0088a5844cf9949 Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 10:57:28 +0300 Subject: [PATCH 06/11] refactor: consolidate migration installer --- README.md | 4 +- docs/migrations/remnashop.md | 8 +- docs/migrations/remnawave-tg-shop.md | 447 ++++++--------------------- scripts/install.sh | 9 +- scripts/migrate_to_minishop.sh | 318 ------------------- tests/test_install_script.py | 3 +- tests/test_migration_doc_accuracy.py | 187 ++++------- 7 files changed, 162 insertions(+), 814 deletions(-) delete mode 100644 scripts/migrate_to_minishop.sh diff --git a/README.md b/README.md index d1c581f..96c9ad9 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,8 @@ Remnawave Minishop - Telegram-бот и Web App (Mini App) для продажи - [Telegram-авторизация](docs/features/telegram-auth.md) и [вход по email](docs/features/email-login.md) - настройка BotFather/OAuth и SMTP-логина. - [Поддержка пользователей / тикеты](docs/features/support.md) - тикеты в Mini App, входящий список админки, уведомления, лимиты и внешняя ссылка поддержки. - [Темы Web App](docs/features/webapp-themes.md) - кастомные темы, настройка внешнего вида, логотипы, CSS/ассеты и пайплайн создания новой темы. -- [Миграции](docs/migrations/index.md) - готовые сценарии переноса с других ботов; сейчас описан `remnawave-tg-shop`. -- [Миграция с remnawave-tg-shop](docs/migrations/remnawave-tg-shop.md) - готовый сценарий для legacy-стека. +- [Миграции](docs/migrations/index.md) - готовые сценарии переноса с `remnawave-tg-shop` и Remnashop. +- [Миграция с remnawave-tg-shop](docs/migrations/remnawave-tg-shop.md) и [Remnashop](docs/migrations/remnashop.md) - сценарии через общий install wizard. ## Совместимость diff --git a/docs/migrations/remnashop.md b/docs/migrations/remnashop.md index 2387bb9..11e662c 100644 --- a/docs/migrations/remnashop.md +++ b/docs/migrations/remnashop.md @@ -1,6 +1,6 @@ # Миграция из Remnashop -Remnashop импортируется через общий legacy-importer `backend/scripts/import_legacy.py`. +Remnashop импортируется через общий скрипт импорта `backend/scripts/import_legacy.py`. Самый удобный путь - интерактивный install wizard: ```bash @@ -8,8 +8,8 @@ curl -fsSL https://raw.githubusercontent.com/3252a8/remnawave-minishop/main/scri sh install.sh ``` -В меню выберите `Install new stack and run legacy migration` для нового сервера -или `Run legacy migration only`, если compose-папка и `.env` уже готовы. +В меню выберите `Install new stack and run migration` для нового сервера +или `Run migration only`, если compose-папка и `.env` уже готовы. ## Что переносится @@ -21,7 +21,7 @@ sh install.sh - служебные mappings, чтобы повторный запуск мог работать в режиме `merge`; - настройки совместимости Remnashop в админке: старые ref-ссылки и promo codes. -Данные, которые не имеют прямого аналога, сохраняются в legacy mappings или +Данные, которые не имеют прямого аналога, сохраняются в служебных таблицах миграции или message logs как заметки, чтобы администратор мог проверить их после переноса. ## Flow wizard diff --git a/docs/migrations/remnawave-tg-shop.md b/docs/migrations/remnawave-tg-shop.md index 789fa71..640336f 100644 --- a/docs/migrations/remnawave-tg-shop.md +++ b/docs/migrations/remnawave-tg-shop.md @@ -1,117 +1,60 @@ -# Миграция с `remnawave-tg-shop` (≤ v2.7.0) на `remnawave-minishop` (v3.4+) +# Миграция с `remnawave-tg-shop` на `remnawave-minishop` -Эта страница - готовый сценарий для legacy-стека `remnawave-tg-shop`. Это единственная миграция с другого бота, которая сейчас описана в документации. Для других Telegram-ботов, самописных панелей и ручных таблиц готового сценария пока нет: их нельзя переносить по этой инструкции без отдельного анализа схемы БД, тарифов, платежей и связи с Remnawave Panel. - -Автоматический скрипт ниже рассчитан именно на родственный стек `remnawave-tg-shop`, где структура БД и Docker volumes известны заранее. Для других ботов нужен отдельный адаптер экспорта/импорта. - -## Новый install wizard - -Для нового сервера или переноса без клонирования репозитория используйте общий -`sh` wizard: +Для переноса со старого родственного стека используйте общий install wizard: ```bash curl -fsSL https://raw.githubusercontent.com/3252a8/remnawave-minishop/main/scripts/install.sh -o install.sh sh install.sh ``` -В меню выберите `Install new stack and run legacy migration` или -`Run legacy migration only`, затем источник `Old remnawave-tg-shop`. -Wizard поддерживает два режима: +В меню выберите `Install new stack and run migration` для нового +сервера или `Run migration only`, если compose-папка уже готова. Затем +выберите источник `Old remnawave-tg-shop`. -- `Copy old Docker volumes` - тот же безопасный сценарий, что и старый - `migrate_to_minishop.sh`: старый volume `remnawave-tg-shop-db-data` - копируется в `remnawave-minishop-db-data`, затем новый stack запускает - сервис `migrate` и накатывает все схемные миграции; -- `Dump from a source PostgreSQL DSN` - новый режим для случаев, когда старая - БД доступна как внешний PostgreSQL DSN. Wizard поднимает целевой `postgres`, - сбрасывает целевую БД, делает `pg_dump` из старой БД, восстанавливает дамп в - compose-БД и затем запускает `migrate`. +Wizard поддерживает два способа переноса: -Старый helper `scripts/migrate_to_minishop.sh` ниже всё ещё полезен для -in-place обновления уже клонированного репозитория: он переключает git-ветку, -переносит volumes и стартует новый stack. Прямой source DSN он не поддерживал. +- `Copy old Docker volumes` - для старого compose-стека на том же Docker host. + Скрипт подготавливает новый stack, копирует + `remnawave-tg-shop-db-data` в `remnawave-minishop-db-data`, опционально + переносит Caddy volumes и запускает новый stack. +- `Dump from a source PostgreSQL DSN` - для старой БД, доступной по DSN. + Скрипт поднимает целевой `postgres`, сбрасывает целевую БД, делает + `pg_dump` из старой БД, восстанавливает дамп в compose-БД и запускает + сервис `migrate`. -## Короткий путь без смены ветки и сборки +В обоих режимах старые volumes и старая БД не удаляются автоматически. -Если вы используете только готовые Docker-образы и не собираете проект -локально, git-команды из ручного способа не нужны. Достаточно обновить -compose-файл до одного из готовых примеров в `deploy/examples` и -перенести/обновить БД. Самый прямой вариант без встроенного обратного прокси - -`deploy/examples/no-proxy/docker-compose.yml`; для Caddy, Nginx и Newt есть -такие же самостоятельные папки. +## Как работает перенос -Минимальная последовательность: +`remnawave-tg-shop` и `remnawave-minishop` имеют совместимую историю схемы. +После переноса старой PostgreSQL-БД сервис `migrate` накатывает недостающие +миграции из `backend/db/migrator.py`: сначала применяются `Base.metadata`, +затем последовательные записи `schema_migrations`. Это one-shot сервис: он +должен завершиться с кодом `0`, после чего стартуют `backend` и `worker`. -```bash -docker compose down +При volume-миграции wizard: -# Скопируйте старый .env в выбранную папку примера и обновите значения там. -cp .env deploy/examples/no-proxy/.env -nano deploy/examples/no-proxy/.env +1. Останавливает известные контейнеры старого и переходного стеков, если вы + подтверждаете этот шаг. +2. Запускает `docker compose up --no-start`, чтобы Docker Compose создал новые + volumes. +3. Копирует старый volume БД: -# Подготовьте стек из готовых образов. -IMAGE_TAG=3.4.0 docker compose \ - --env-file deploy/examples/no-proxy/.env \ - -f deploy/examples/no-proxy/docker-compose.yml \ - up --no-start + ```bash + docker run --rm \ + -v remnawave-tg-shop-db-data:/from:ro \ + -v remnawave-minishop-db-data:/to \ + alpine sh -c "cd /from && cp -a . /to" + ``` -# Нужно только при переходе со старого имени volume remnawave-tg-shop-db-data. -# Если у вас уже есть remnawave-minishop-db-data, этот шаг пропустите. -docker run --rm \ - -v remnawave-tg-shop-db-data:/from:ro \ - -v remnawave-minishop-db-data:/to \ - alpine sh -c "cd /from && cp -a . /to" +4. Если старые Caddy volumes существуют, переносит + `remnawave-tg-shop-caddy-data` -> `remnawave-minishop-caddy-data` и + `remnawave-tg-shop-caddy-config` -> `remnawave-minishop-caddy-config`. +5. Запускает новый stack через Docker Compose. -IMAGE_TAG=3.4.0 docker compose \ - --env-file deploy/examples/no-proxy/.env \ - -f deploy/examples/no-proxy/docker-compose.yml \ - up -d -docker compose \ - --env-file deploy/examples/no-proxy/.env \ - -f deploy/examples/no-proxy/docker-compose.yml \ - logs migrate -``` - -Сервис `migrate` сам применит недостающие схемные миграции к перенесённому -тому PostgreSQL. Новые тома `remnawave-minishop-redis-data` и -`remnawave-minishop-shop-data` переносить не нужно: они создаются пустыми. - -Этот документ описывает обновление стека, поднятого по `remnawave-tg-shop` -(включая последний релиз `v2.7.0` форка `kavore/remnawave-tg-shop`), до -текущей версии `remnawave-minishop` (v3.4+). Между этими версиями произошли -две независимые перетряски, и скрипт пытается отработать обе одной командой: - -1. **Переименование стека** (v3.1.0): контейнеры и тома `remnawave-tg-shop-*` - стали `remnawave-minishop-*`. Простой `docker compose up -d` после - `git pull` создаёт пустую БД — без переноса тома данные теряются. -2. **Разделение бота на сервисы** (v3.4.0): из одного контейнера выделены - `backend`, `worker`, `frontend`, `migrate` + новые `postgres`, `redis`. - Появились новые volumes `redis-data` и `shop-data`, новые обязательные - переменные окружения, а схема БД обновляется автоматически one-shot - сервисом `migrate`. - -После миграции `docker compose ps` должен показать как минимум: `backend`, -`worker`, `frontend`, `postgres`, `redis` (running) и `migrate` (exited 0). -Логи: `docker compose logs -f backend worker frontend`. - -Доступные пути: - -- [Автоматический](#автоматический-способ-через-скрипт) — скрипт-обёртка - останавливает старый стек, накатывает свежий код, переносит том БД, - поднимает новые сервисы. Идемпотентный. -- [Ручной](#ручной-способ) — те же шаги командами, для тех, кому нужно - понимать каждое действие или выполнить выборочно. - -В обоих случаях: - -- старые тома **не удаляются** автоматически — это безопасный бэкап на случай - отката; -- сертификаты Caddy (если используется `deploy/examples/caddy/docker-compose.yml`) - тоже переносятся, чтобы Let's Encrypt не выписывал их заново и не упереться - в rate limit; -- схема БД обновляется автоматически: при первом `docker compose up -d` сервис - `migrate` накатывает на перенесённый том все недостающие миграции (от - alembic-схемы v2.7.0 до текущей). +Если целевой DB volume уже непустой, wizard не перетирает его молча: он +останавливается и просит отдельное подтверждение на продолжение без копирования +старой БД. ## Что меняется в архитектуре @@ -120,256 +63,55 @@ docker compose \ | Версия | Сервисы | | --- | --- | | `v2.7.0` | `remnawave-tg-shop`, `remnawave-tg-shop-db` | -| `v3.1.x–v3.3.x` | `remnawave-minishop`, `remnawave-minishop-db` | -| `v3.4+` (текущая) | `remnawave-minishop-backend`, `remnawave-minishop-worker`, `remnawave-minishop-frontend`, `remnawave-minishop-migrate`, `remnawave-minishop-postgres`, `remnawave-minishop-redis` | - -Внутри Docker-сети сервисы доступны по коротким DNS-именам (`backend`, `worker`, -`frontend`, `postgres`, `redis`), а не по полному `container_name`. Это важно -для внешнего reverse-proxy — см. раздел [Внешний reverse-proxy](#внешний-reverse-proxy) ниже. +| `v3.1.x-v3.3.x` | `remnawave-minishop`, `remnawave-minishop-db` | +| `v3.4+` | `remnawave-minishop-backend`, `remnawave-minishop-worker`, `remnawave-minishop-frontend`, `remnawave-minishop-migrate`, `remnawave-minishop-postgres`, `remnawave-minishop-redis` | **Volumes**: -| Volume | v2.7.0 | v3.4+ | Что внутри | -| --- | --- | --- | --- | -| `remnawave-minishop-db-data` | переименовать из `remnawave-tg-shop-db-data` | переносится скриптом | PostgreSQL | -| `remnawave-minishop-redis-data` | — | создаётся пустым | Redis (FSM, rate-limit, cache, очередь вебхуков, distributed locks) | -| `remnawave-minishop-shop-data` | — | создаётся пустым | `/app/data`: `tariffs.json`, темы Web App, кэш логотипа/emoji | -| `remnawave-minishop-caddy-data` / `remnawave-minishop-caddy-config` | переименовать из `remnawave-tg-shop-caddy-*` | переносится скриптом | только при Caddy-варианте | +| Volume | Что происходит | +| --- | --- | +| `remnawave-minishop-db-data` | переносится из `remnawave-tg-shop-db-data` или восстанавливается из source DSN | +| `remnawave-minishop-redis-data` | создается пустым | +| `remnawave-minishop-shop-data` | создается пустым; runtime-файлы в `/app/data` дальше настраиваются через админку или вручную | +| `remnawave-minishop-caddy-data` / `remnawave-minishop-caddy-config` | переносятся из `remnawave-tg-shop-caddy-*`, если старый стек использовал Caddy | -`redis-data` и `shop-data` стартуют пустыми — это нормально. Redis ничего -долгоживущего не хранит (всё либо FSM, либо кеш с TTL), а `data/` инициализируется -из образа при первом старте (`tariffs.json` пуст пока вы не сконфигурируете -тарифы через админ-панель). +Доступные compose-профили: `docker-compose.yml`, +`deploy/examples/caddy/docker-compose.yml`, +`deploy/examples/nginx/docker-compose.yml`, +`deploy/examples/newt/docker-compose.yml`, +`deploy/examples/no-proxy/docker-compose.yml`. -## Переменные окружения, которые могли исчезнуть или переехать +## Переменные окружения -Перед запуском нового стека проверьте `.env`. Ниже — только то, что точно -менялось между v2.7.0 и v3.4+: +Перед запуском нового стека проверьте `.env`. Самые важные изменения: -| Было (v2.7.0) | Стало (v3.4+) | Действие | +| Было | Стало | Действие | | --- | --- | --- | -| `TELEGRAM_WEBHOOK_SECRET` | `WEBHOOK_SECRET_TOKEN` | Переименовать. Если пусто — будет сгенерирован при старте, но тогда Telegram переустановит webhook (на это не реагирует существующий запрос). | -| `TELEGRAM_WEBHOOK_PATH` | удалена | Путь вебхука теперь генерируется из `BOT_TOKEN` автоматически. | -| `REQUIRED_CHANNEL_SUBSCRIBE_TO_USE` | удалена | Гейт включается автоматически, как только задан `REQUIRED_CHANNEL_ID`. | -| `STARS_PROVIDER_TOKEN` | удалена | Telegram Stars (XTR) используются напрямую. | -| `REFERRAL_ENABLED` | удалена | Реферальная программа активна по умолчанию. В legacy-режиме без JSON-каталога отключайте платежные бонусы через нули в `REFERRAL_BONUS_DAYS_*` и `REFEREE_BONUS_DAYS_*`; в JSON-тарифах обнуляйте или удаляйте `referral_bonus_days_inviter` и `referral_bonus_days_referee` у period-тарифов. | -| `POSTGRES_HOST=remnawave-tg-shop-db` | в `.env` — `remnawave-minishop-db` или пусто | Под compose значение всё равно переопределяется на сервисное имя `postgres` (см. `environment:` в compose-файлах), поэтому скрипт правит `.env` только для bare-metal сценариев. | -| `WEBHOOK_BASE_URL` | **обязательна** | Polling-режим удалён, без публичного URL бот не стартует. | -| — | `REDIS_URL=redis://redis:6379/0` | Обязательна для воркера, очередей и rate-limit. По умолчанию в compose-файлах уже задана. | -| — | `WEBAPP_SESSION_SECRET`, `WEBAPP_ENABLED`, `WEBAPP_SERVER_PORT`, `WEBAPP_THEMES_DIR`, `TARIFFS_CONFIG_PATH` | Новые настройки Web App / тарифного каталога. Безопасные дефолты есть в `.env.example`. | +| `TELEGRAM_WEBHOOK_SECRET` | `WEBHOOK_SECRET_TOKEN` | Перенести значение или сгенерировать новый stable secret. | +| `TELEGRAM_WEBHOOK_PATH` | удалена | Путь вебхука теперь рассчитывается автоматически. | +| `REQUIRED_CHANNEL_SUBSCRIBE_TO_USE` | удалена | Гейт включается, когда задан `REQUIRED_CHANNEL_ID`. | +| `STARS_PROVIDER_TOKEN` | удалена | Telegram Stars используются напрямую. | +| `POSTGRES_HOST=remnawave-tg-shop-db` | `postgres` внутри Compose | В compose-файлах `POSTGRES_HOST` переопределяется service name `postgres`. | +| `WEBHOOK_BASE_URL` | обязательна | Без публичного URL backend не стартует корректно. | +| - | `REDIS_URL=redis://redis:6379/0` | В compose-профилях задано автоматически. | +| - | `WEBAPP_SESSION_SECRET`, `WEBAPP_ENABLED`, `TARIFFS_CONFIG_PATH` | Новые настройки Web App и каталога тарифов. | -Полный референс — [docs/getting-started/configuration.md](../getting-started/configuration.md). Скрипт миграции -эти переменные **не правит** автоматически (только `POSTGRES_HOST`), потому -что у каждой инсталляции свой шаблон `.env` с кастомными значениями. Лучше -сравнить свой `.env` с `.env.example` глазами один раз, чем получить -несовместимый шаблон автоматом. +Остальные продуктовые настройки удобнее проверить после первого входа в +админку. -## Автоматический способ (через скрипт) +## Reverse Proxy -Если helper ещё не лежит у вас локально, запускайте его прямо из `raw` из -корня старого репозитория: +В старом стеке часто был один upstream `remnawave-tg-shop:8000`. В текущем +split-arch stack маршруты разделены: -```bash -bash <(curl -fsSL https://raw.githubusercontent.com/3252a8/remnawave-minishop/main/scripts/migrate_to_minishop.sh) -``` - -> Команда выше рассчитана на `bash` / Git Bash / WSL. Если вы запускаете из -> PowerShell, удобнее сначала открыть Git Bash. - -Если вы уже подтянули новую версию и файл есть локально, можно запускать так: - -```bash -bash scripts/migrate_to_minishop.sh -``` - -По умолчанию скрипт работает с `docker-compose.yml` и переключается на ветку -`main`. Можно переопределить через переменные окружения: - -| Переменная | Назначение | По умолчанию | -| ----------------- | ----------------------------------------------------------------------- | ---------------------- | -| `PROJECT_ROOT` | Явный путь к корню старого репозитория, если запуск не из него | текущая директория | -| `COMPOSE_FILE` | Какой compose-файл стартовать в конце | `docker-compose.yml` | -| `TARGET_BRANCH` | На какую ветку переключаться и подтягивать обновления | `main` | -| `GIT_REMOTE` | Какой remote использовать для `fetch`/`pull` | `origin` | -| `NEW_ORIGIN_URL` | Если задано и не совпадает с URL выбранного remote — он будет обновлён | (не меняется) | -| `ASSUME_YES` | `1` — не задавать интерактивных вопросов | `0` | - -Примеры: - -```bash -# Caddy-вариант из raw-файла. -# Перед запуском скопируйте старый .env в deploy/examples/caddy/.env -# и заполните WEBHOOK_HOST / MINIAPP_HOST. -COMPOSE_FILE=deploy/examples/caddy/docker-compose.yml \ - bash <(curl -fsSL https://raw.githubusercontent.com/3252a8/remnawave-minishop/main/scripts/migrate_to_minishop.sh) - -# С переключением origin на форк 3252a8 -NEW_ORIGIN_URL=https://github.com/3252a8/remnawave-minishop.git \ - bash <(curl -fsSL https://raw.githubusercontent.com/3252a8/remnawave-minishop/main/scripts/migrate_to_minishop.sh) - -# Без интерактива -ASSUME_YES=1 \ - bash <(curl -fsSL https://raw.githubusercontent.com/3252a8/remnawave-minishop/main/scripts/migrate_to_minishop.sh) -``` - -Что делает скрипт: - -1. **Останавливает текущий стек**: ищет известные контейнеры старой схемы - (`remnawave-tg-shop`, `…-db`, `…-caddy`), переходного периода - (`remnawave-minishop`, `…-db`, `…-caddy`) и новой схемы - (`…-backend`, `…-worker`, `…-frontend`, `…-migrate`, `…-postgres`, `…-redis`) - и останавливает их, если запущены. Безопасно при повторном запуске. -2. **Переключает `origin`**, если задана переменная `NEW_ORIGIN_URL`, иначе - оставляет как есть. -3. **Подтягивает целевую ветку** (`git fetch` + `git switch` + `git pull --ff-only`). - Прерывается, если в рабочем дереве есть незакоммиченные изменения. -4. **Правит `POSTGRES_HOST` в `.env`** (только для bare-metal сценариев — в - compose это значение перебивает `environment:` блок). -5. **Подготавливает новый стек в режиме `--no-start`**, чтобы Compose сам - создал тома `db-data`, `redis-data`, `shop-data` и не ругался на уже - существующий volume. -6. **Переносит том БД** `remnawave-tg-shop-db-data` → `remnawave-minishop-db-data` - (и Caddy-тома, если применимо) через одноразовый `alpine`-контейнер. Если - новый том уже непустой — копирование пропускается. Новые volumes - `redis-data` и `shop-data` остаются пустыми (их и не должно быть в старом - стеке). -7. **Стартует новый стек** (`docker compose up -d --remove-orphans` плюс - `--build` для локальной сборки). `migrate` отработает первым, накатит - на перенесённый том все недостающие миграции (от alembic-схемы v2.7.0 до - текущей) и завершится. Затем стартуют `backend`, `worker`, `frontend`. - -Скрипт идемпотентен: повторный запуск ничего не сломает, просто пропустит уже -выполненные шаги. - -После того как убедитесь, что бот работает и данные на месте, удалите старые -тома: - -```bash -docker volume rm remnawave-tg-shop-db-data -docker volume rm remnawave-tg-shop-caddy-data remnawave-tg-shop-caddy-config 2>/dev/null || true -``` - -## Ручной способ - -1. **Остановите старый стек и обновите код:** - - ```bash - docker compose down - git fetch origin - git checkout main - git pull --ff-only origin main - ``` - -2. **(Только для bare-metal без compose)** обновите `.env`, если в нём ещё - жёстко прописан старый контейнер БД: - - ```bash - sed -i.bak 's/^POSTGRES_HOST=remnawave-tg-shop-db$/POSTGRES_HOST=remnawave-minishop-db/' .env - ``` - - Под `docker compose up` это не нужно: compose сам выставляет - `POSTGRES_HOST: postgres` (имя сервиса) в `environment:` и `.env`-значение - не используется. - -3. **Проверьте `.env`** на наличие переменных, которые исчезли или - переименовались — см. раздел - [Переменные окружения](#переменные-окружения-которые-могли-исчезнуть-или-переехать) - выше. Главное: `WEBHOOK_SECRET_TOKEN` (бывший `TELEGRAM_WEBHOOK_SECRET`), - обязательный `WEBHOOK_BASE_URL` и наличие `REDIS_URL` (по умолчанию задано - в compose). - -4. **Подготовьте новый стек без запуска**, чтобы Compose создал новые volumes - (`db-data`, `redis-data`, `shop-data`) и контейнеры: - - ```bash - # Локальная сборка - docker compose up --no-start --build - - # Или готовый Caddy-вариант из GHCR-образов - cp .env deploy/examples/caddy/.env - nano deploy/examples/caddy/.env - docker compose \ - --env-file deploy/examples/caddy/.env \ - -f deploy/examples/caddy/docker-compose.yml \ - up --no-start - - # Другие готовые варианты: - # deploy/examples/nginx/docker-compose.yml - # deploy/examples/newt/docker-compose.yml - # deploy/examples/no-proxy/docker-compose.yml - ``` - -5. **Перенесите том БД в новое имя:** - - ```bash - docker run --rm \ - -v remnawave-tg-shop-db-data:/from:ro \ - -v remnawave-minishop-db-data:/to \ - alpine sh -c "cd /from && cp -a . /to" - ``` - - `remnawave-minishop-redis-data` и `remnawave-minishop-shop-data` — новые, - переносить нечего. Они инициализируются на лету: Redis пуст, а `data/` - наполняется при первом обращении к настройкам Web App / каталогу тарифов. - -6. **(Только для Caddy)** перенесите тома Caddy с TLS-сертификатами и - состоянием ACME: - - ```bash - for v in caddy-data caddy-config; do - docker run --rm \ - -v "remnawave-tg-shop-$v":/from:ro \ - -v "remnawave-minishop-$v":/to \ - alpine sh -c "cd /from && cp -a . /to" - done - ``` - -7. **Запустите новый стек:** - - ```bash - docker compose up -d - # или - docker compose \ - --env-file deploy/examples/caddy/.env \ - -f deploy/examples/caddy/docker-compose.yml \ - up -d - ``` - - Сервис `migrate` запустится первым, обнаружит перенесённый том, - применит недостающие схемные миграции (`Base.metadata.create_all` + - последовательные миграции `0001..00NN` из `backend/db/migrator.py`) и - выйдет с кодом 0. Только после этого стартуют `backend` и `worker`. - -8. **Проверьте состояние:** - - ```bash - docker compose ps - docker compose logs -f backend worker frontend - docker compose logs migrate # должен закончиться "Migrator: migration 00NN applied successfully" - ``` - -9. **(Опционально) удалите старые тома**, когда убедитесь, что новый стек - стабилен: - - ```bash - docker volume rm remnawave-tg-shop-db-data - docker volume rm remnawave-tg-shop-caddy-data remnawave-tg-shop-caddy-config 2>/dev/null || true - ``` - -## Внешний reverse-proxy - -В v2.7.0 был один upstream — `remnawave-tg-shop:8000`. В v3.4+ функциональность -разнесена по портам и сервисам: - -| Назначение | DNS-имя сервиса | Порт | +| Назначение | Service | Port | | --- | --- | --- | -| Telegram / платежные / вебхуки панели | `backend` | `8080` | -| Health-чек | `backend` | `8080` (`/healthz`) | -| Web App API (`/api/*`, `/auth/*`, ассеты тем и логотипов) | `backend` | `8081` (доступен только из Docker-сети) | -| Статический фронт Web App | `frontend` | `80` (внутри `frontend` уже проксирует `/api/*` и `/auth/*` на `backend:8081`) | +| Telegram, платежные и panel webhooks | `backend` | `8080` | +| Health-check | `backend` | `8080` (`/healthz`) | +| Web App API и auth | `backend` | `8081` внутри Docker-сети | +| Статический Web App frontend | `frontend` | `80` | -Минимальная замена для внешнего Nginx, который раньше слал всё на один -upstream: +Минимальная схема для внешнего Nginx: ```nginx upstream remnawave_backend_webhooks { server backend:8080; } @@ -378,33 +120,32 @@ upstream remnawave_frontend { server frontend:80; } server { server_name app.domain.com; listen 443 ssl; - http2 on; - # ssl_certificate / ssl_certificate_key — без изменений location /webhook/ { proxy_pass http://remnawave_backend_webhooks; } location /healthz { proxy_pass http://remnawave_backend_webhooks; } - location / { proxy_pass http://remnawave_frontend; } + location / { proxy_pass http://remnawave_frontend; } } ``` -Полные примеры (Caddy, Nginx, Newt/Pangolin и запуск без обратного прокси) — в -[docs/getting-started/deployment.md](../getting-started/deployment.md) и [docs/features/web-app.md](../features/web-app.md). Если раньше прокси указывал на -`remnawave-tg-shop:8000` напрямую, после миграции нужно либо переключиться на -`backend:8080` / `frontend:80`, либо использовать готовый Caddy/Nginx/Newt -пример, который уже знает правильную маршрутизацию. +Готовые Caddy, Nginx, Pangolin/Newt и no-proxy профили уже содержат нужную +маршрутизацию. -## Если что-то пошло не так +## Проверка -`migrate` упал → читайте `docker compose logs migrate`. Том БД остался -не тронут, можно откатиться, переключив compose-файл обратно на старый -коммит и подняв старый стек на старом томе `remnawave-tg-shop-db-data` -(пока вы его не удалили). +После переноса: -`backend` не стартует → чаще всего `WEBHOOK_BASE_URL` пуст, либо -`WEBHOOK_SECRET_TOKEN` отличается от того, что Telegram ждёт. Поставьте -свежий секрет в `.env` и перезапустите — Telegram переустановит webhook -автоматически. +```bash +docker compose ps +docker compose logs migrate +docker compose logs -f backend worker frontend +``` -Web App пуст / 502 → проверьте, что `frontend` живёт (`docker compose ps`), -а внешний прокси шлёт на `frontend:80`, а не на старый -`remnawave-tg-shop:8000`. +`migrate` должен завершиться успешно, а `backend`, `worker`, `frontend`, +`postgres` и `redis` должны быть running/healthy. + +Когда убедитесь, что новый stack работает, старые volumes можно удалить вручную: + +```bash +docker volume rm remnawave-tg-shop-db-data +docker volume rm remnawave-tg-shop-caddy-data remnawave-tg-shop-caddy-config 2>/dev/null || true +``` diff --git a/scripts/install.sh b/scripts/install.sh index 26274be..95b7835 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -939,7 +939,7 @@ run_remnashop_migration() { if confirm "Restart backend and worker so setting overrides are reloaded?" 1; then (cd "$TARGET_DIR" && run_compose restart backend worker) || true fi - ok "Legacy migration completed." + ok "Migration completed." } run_target_schema_migrations() { @@ -980,11 +980,10 @@ run_tgshop_volume_migration() { run_tgshop_dsn_migration() { section "Old remnawave-tg-shop DSN migration" - warn "The old standalone helper did not support direct DSN import." warn "This wizard path dumps the old PostgreSQL database, restores it into target Compose PostgreSQL, then runs Minishop schema migrations." warn "The target database will be dropped and recreated before restore." - if ! confirm "Replace target database with the legacy dump?" 0; then + if ! confirm "Replace target database with the source dump?" 0; then warn "Migration not applied." return 0 fi @@ -1141,8 +1140,8 @@ main_menu() { banner choose "Main menu" "1" "1|2|3|4|5|6" \ "1. Install new stack" \ - "2. Install new stack and run legacy migration" \ - "3. Run legacy migration only" \ + "2. Install new stack and run migration" \ + "3. Run migration only" \ "4. Download/update deployment files only" \ "5. Validate current stack" \ "6. Exit" diff --git a/scripts/migrate_to_minishop.sh b/scripts/migrate_to_minishop.sh deleted file mode 100644 index b8c8546..0000000 --- a/scripts/migrate_to_minishop.sh +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="" - -OLD_PREFIX="remnawave-tg-shop" -NEW_PREFIX="remnawave-minishop" -OLD_DB_VOLUME="${OLD_PREFIX}-db-data" -NEW_DB_VOLUME="${NEW_PREFIX}-db-data" -OLD_CADDY_VOLUMES=("${OLD_PREFIX}-caddy-data" "${OLD_PREFIX}-caddy-config") -NEW_CADDY_VOLUMES=("${NEW_PREFIX}-caddy-data" "${NEW_PREFIX}-caddy-config") -# Container names span three eras: original ``remnawave-tg-shop*`` (≤ v2.7.0), -# the renamed but still single-container ``remnawave-minishop*`` (v3.1.0 – -# v3.3.x), and the split-arch stack introduced in v3.4.0 (backend / worker / -# frontend / migrate / postgres / redis, plus optional caddy). The list is -# used only to stop existing containers before migration, so it is safe — -# and idempotent — to include every known name from every era. -KNOWN_CONTAINERS=( - # v2.7.0 (upstream remnawave-tg-shop): - "${OLD_PREFIX}" - "${OLD_PREFIX}-db" - "${OLD_PREFIX}-caddy" - # v3.1.0 – v3.2.x (renamed but still one container): - "${NEW_PREFIX}" - "${NEW_PREFIX}-db" - "${NEW_PREFIX}-caddy" - # v3.4.0+ (split architecture): - "${NEW_PREFIX}-backend" - "${NEW_PREFIX}-worker" - "${NEW_PREFIX}-frontend" - "${NEW_PREFIX}-migrate" - "${NEW_PREFIX}-postgres" - "${NEW_PREFIX}-redis" -) - -log() { - printf '%s\n' "$*" -} - -die() { - printf 'Ошибка: %s\n' "$*" >&2 - exit 1 -} - -require_cmd() { - command -v "$1" >/dev/null 2>&1 || die "Не найдено обязательное средство \`$1\` в PATH." -} - -resolve_root() { - if [[ -n "${PROJECT_ROOT:-}" ]]; then - [[ -d "$PROJECT_ROOT" ]] || die "PROJECT_ROOT не существует: $PROJECT_ROOT" - (cd -- "$PROJECT_ROOT" >/dev/null && pwd -P) - return - fi - - local git_root - if git_root="$(git rev-parse --show-toplevel 2>/dev/null)"; then - printf '%s\n' "$git_root" - return - fi - - pwd -P -} - -run() { - log "+ $*" - "$@" -} - -container_exists() { - docker inspect "$1" >/dev/null 2>&1 -} - -container_running() { - [[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || true)" == "true" ]] -} - -stop_container() { - local name="$1" - - if ! container_exists "$name"; then - return 1 - fi - - if container_running "$name"; then - docker stop "$name" >/dev/null - fi - docker rm "$name" >/dev/null -} - -volume_exists() { - docker volume inspect "$1" >/dev/null 2>&1 -} - -volume_is_empty() { - docker run --rm -v "$1:/data" alpine sh -c 'test -z "$(find /data -mindepth 1 -print -quit)"' >/dev/null 2>&1 -} - -copy_volume() { - local source="$1" - local target="$2" - - if ! volume_exists "$source"; then - log " - Пропускаю том \`$source\`: исходный том не найден." - return 1 - fi - - if volume_exists "$target" && ! volume_is_empty "$target"; then - log " - Пропускаю том \`$target\`: он уже не пустой." - return 1 - fi - - if ! volume_exists "$target"; then - die "Целевой том \`$target\` не создан Compose. Сначала нужно подготовить новый стек в режиме \`--no-start\`." - fi - - docker run --rm -v "$source:/from:ro" -v "$target:/to" alpine sh -c 'cd /from && cp -a . /to/' -} - -is_old_postgres_host() { - grep -Eq "^[[:space:]]*POSTGRES_HOST[[:space:]]*=[[:space:]]*${OLD_PREFIX}-db[[:space:]]*(#.*)?$" "$ROOT/.env" -} - -is_new_postgres_host() { - grep -Eq "^[[:space:]]*POSTGRES_HOST[[:space:]]*=[[:space:]]*${NEW_PREFIX}-db[[:space:]]*(#.*)?$" "$ROOT/.env" -} - -update_postgres_host() { - if is_old_postgres_host; then - sed -i.bak -E "s|^([[:space:]]*POSTGRES_HOST[[:space:]]*=[[:space:]]*)${OLD_PREFIX}-db([[:space:]]*(#.*)?)$|\\1${NEW_PREFIX}-db\\2|" "$ROOT/.env" - log " - \`.env\` обновлён, резервная копия сохранена в \`.env.bak\`." - elif is_new_postgres_host; then - log " - \`POSTGRES_HOST\` уже указывает на новый контейнер, ничего менять не нужно." - else - log " - \`POSTGRES_HOST\` не похож на старую схему, пропускаю изменение." - fi -} - -main() { - require_cmd git - require_cmd docker - docker info >/dev/null - - ROOT="$(resolve_root)" - - local compose_file="${COMPOSE_FILE:-docker-compose.yml}" - local target_branch="${TARGET_BRANCH:-main}" - local git_remote="${GIT_REMOTE:-origin}" - local new_origin_url="${NEW_ORIGIN_URL:-}" - local assume_yes="${ASSUME_YES:-0}" - local current_origin - local current_branch - local remote_ref - local head_commit - local compose_has_build=0 - local compose_has_caddy=0 - local -a compose_cmd - local -a running_containers=() - local -a summary=() - local -a up_args - local name - local source - local target - local answer - - if [[ $compose_file != /* ]]; then - compose_file="$ROOT/$compose_file" - fi - [[ -f "$compose_file" ]] || die "Compose-файл не найден: $compose_file" - [[ -e "$ROOT/.git" ]] || die "Скрипт нужно запускать из корня git-репозитория." - [[ -f "$ROOT/.env" ]] || die "Не найден \`.env\` в корне репозитория." - - if docker compose version >/dev/null 2>&1; then - compose_cmd=(docker compose) - elif command -v docker-compose >/dev/null 2>&1; then - compose_cmd=(docker-compose) - else - die "Не найден ни \`docker compose\`, ни \`docker-compose\`." - fi - - if [[ -n "$(git -C "$ROOT" status --porcelain=v1)" ]]; then - die "В рабочем дереве есть незакоммиченные изменения. Сначала сохраните их, чтобы миграция не затёрла чужие правки." - fi - - if grep -Eq '^[[:space:]]*build:[[:space:]]*' "$compose_file"; then - compose_has_build=1 - fi - if grep -Eq '^[[:space:]]*caddy:[[:space:]]*$' "$compose_file"; then - compose_has_caddy=1 - fi - - for name in "${KNOWN_CONTAINERS[@]}"; do - if container_exists "$name"; then - running_containers+=("$name") - fi - done - - current_origin="$(git -C "$ROOT" remote get-url "$git_remote")" - if [[ -n "$new_origin_url" && "$current_origin" != "$new_origin_url" ]]; then - summary+=( "обновить $git_remote с \`$current_origin\` на \`$new_origin_url\`" ) - fi - summary+=( "скачать ветку \`$target_branch\` из \`$git_remote\`" ) - if volume_exists "$OLD_DB_VOLUME"; then - summary+=( "проверить том БД \`$OLD_DB_VOLUME\` и перенести в \`$NEW_DB_VOLUME\` при необходимости" ) - fi - if ((compose_has_caddy)); then - for i in 0 1; do - source="${OLD_CADDY_VOLUMES[$i]}" - target="${NEW_CADDY_VOLUMES[$i]}" - if volume_exists "$source"; then - summary+=( "проверить том \`$source\` и перенести в \`$target\` при необходимости" ) - fi - done - fi - if is_old_postgres_host; then - summary+=( "обновить \`POSTGRES_HOST\` в \`.env\`" ) - fi - summary+=( "подготовить новый стек через Compose в режиме \`--no-start\`" ) - summary+=( "запустить compose-файл \`$(basename "$compose_file")\`" ) - - if [[ "$assume_yes" != "1" ]]; then - if [[ ! -t 0 ]]; then - die "Скрипт ожидает интерактивное подтверждение. Запустите с \`ASSUME_YES=1\` для неинтерактивного режима." - fi - log "План миграции:" - for name in "${summary[@]}"; do - log " - $name" - done - read -r -p "Продолжить? [y/N]: " answer - case "$answer" in - y|Y|yes|YES|Yes) - ;; - *) - die "Миграция отменена пользователем." - ;; - esac - fi - - log "1. Останавливаю старый стек" - if ((${#running_containers[@]})); then - for name in "${running_containers[@]}"; do - stop_container "$name" - log " - контейнер \`$name\` остановлен/удалён" - done - else - log " - запущенных контейнеров старой схемы не найдено" - fi - - if [[ -n "$new_origin_url" && "$current_origin" != "$new_origin_url" ]]; then - log "2. Обновляю origin" - run git -C "$ROOT" remote set-url "$git_remote" "$new_origin_url" - else - log "2. Origin уже актуален, пропускаю" - fi - - log "3. Обновляю git до ветки \`$target_branch\`" - run git -C "$ROOT" fetch "$git_remote" "$target_branch" - - current_branch="$(git -C "$ROOT" branch --show-current || true)" - if [[ -z "$current_branch" ]]; then - current_branch="$(git -C "$ROOT" rev-parse --abbrev-ref HEAD)" - fi - - if [[ "$current_branch" != "$target_branch" ]]; then - if git -C "$ROOT" show-ref --verify --quiet "refs/heads/$target_branch"; then - run git -C "$ROOT" switch "$target_branch" - else - run git -C "$ROOT" switch -c "$target_branch" --track "$git_remote/$target_branch" - fi - else - log " - уже на ветке \`$target_branch\`" - fi - - remote_ref="$(git -C "$ROOT" rev-parse "$git_remote/$target_branch")" - head_commit="$(git -C "$ROOT" rev-parse HEAD)" - if [[ "$head_commit" != "$remote_ref" ]]; then - run git -C "$ROOT" pull --ff-only "$git_remote" "$target_branch" - else - log " - локальная ветка уже совпадает с удалённой, \`git pull\` не нужен" - fi - - log "4. Обновляю \`.env\`" - update_postgres_host - - log "5. Подготавливаю новый стек через Compose" - if ((compose_has_build)); then - run "${compose_cmd[@]}" -f "$compose_file" up --no-start --build - else - run "${compose_cmd[@]}" -f "$compose_file" up --no-start - fi - - log "6. Переношу тома" - if copy_volume "$OLD_DB_VOLUME" "$NEW_DB_VOLUME"; then - log " - БД перенесена в \`$NEW_DB_VOLUME\`" - fi - if ((compose_has_caddy)); then - for i in 0 1; do - source="${OLD_CADDY_VOLUMES[$i]}" - target="${NEW_CADDY_VOLUMES[$i]}" - if copy_volume "$source" "$target"; then - log " - \`$source\` перенесён в \`$target\`" - fi - done - fi - - log "7. Запускаю новый стек" - if ((compose_has_build)); then - up_args=(up -d --build --remove-orphans) - else - up_args=(up -d --remove-orphans) - fi - run "${compose_cmd[@]}" -f "$compose_file" "${up_args[@]}" - run "${compose_cmd[@]}" -f "$compose_file" ps - - log "Готово." -} - -main "$@" diff --git a/tests/test_install_script.py b/tests/test_install_script.py index 72a3b56..654627e 100644 --- a/tests/test_install_script.py +++ b/tests/test_install_script.py @@ -37,7 +37,8 @@ def test_shell_installer_downloads_raw_files_and_runs_import_in_container(): assert "git clone" not in script assert "backend python backend/scripts/import_legacy.py" in script assert "--dry-run" in script - assert "Install new stack and run legacy migration" in script + assert "Install new stack and run migration" in script + assert "Run migration only" in script def test_shell_installer_supports_legacy_tgshop_volume_and_dsn_paths(): diff --git a/tests/test_migration_doc_accuracy.py b/tests/test_migration_doc_accuracy.py index 30503c7..5dd79b7 100644 --- a/tests/test_migration_doc_accuracy.py +++ b/tests/test_migration_doc_accuracy.py @@ -1,25 +1,15 @@ -"""Pin facts that ``docs/migrations/remnawave-tg-shop.md`` and -``scripts/migrate_to_minishop.sh`` rely on. - -Both documents are written for a user upgrading from ``remnawave-tg-shop`` -(v2.7.0 era) to the current split-arch ``remnawave-minishop`` (v3.4+). They -make concrete claims about: - -* the set of container names produced by today's compose files; -* the set of volume names produced by today's compose files; -* the eras the migration script is allowed to stop containers from. - -If any of these drift apart from reality, the migration document silently -goes stale. These tests fail loudly instead. -""" +"""Pin facts shared by the migration docs and the unified install wizard.""" import re +import shutil +import subprocess import unittest from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] DOC_PATH = REPO_ROOT / "docs" / "migrations" / "remnawave-tg-shop.md" -SCRIPT_PATH = REPO_ROOT / "scripts" / "migrate_to_minishop.sh" +INSTALL_SCRIPT_PATH = REPO_ROOT / "scripts" / "install.sh" +REMOVED_SCRIPT_PATH = REPO_ROOT / "scripts" / "migrate_to_minishop.sh" COMPOSE_FILES = ( REPO_ROOT / "docker-compose.yml", REPO_ROOT / "deploy" / "examples" / "caddy" / "docker-compose.yml", @@ -28,7 +18,6 @@ COMPOSE_FILES = ( REPO_ROOT / "deploy" / "examples" / "no-proxy" / "docker-compose.yml", ) -# Names that the current architecture must produce in at least one compose file. EXPECTED_CONTAINER_NAMES = { "remnawave-minishop-backend", "remnawave-minishop-worker", @@ -52,15 +41,27 @@ def _all_compose_text() -> str: return "\n".join(_read(path) for path in COMPOSE_FILES if path.is_file()) +def _known_containers_from_install_script() -> set[str]: + text = _read(INSTALL_SCRIPT_PATH) + match = re.search(r'^KNOWN_LEGACY_CONTAINERS="([^"]+)"', text, flags=re.MULTILINE) + if not match: + return set() + return set(match.group(1).split()) + + class MigrationDocumentationFactsTests(unittest.TestCase): def setUp(self) -> None: self.doc = _read(DOC_PATH) - self.script = _read(SCRIPT_PATH) + self.script = _read(INSTALL_SCRIPT_PATH) self.compose = _all_compose_text() + def test_unified_install_script_is_the_only_migration_entrypoint(self): + self.assertTrue(INSTALL_SCRIPT_PATH.is_file()) + self.assertFalse(REMOVED_SCRIPT_PATH.exists()) + self.assertIn("scripts/install.sh", self.doc) + self.assertNotIn("migrate_to_minishop.sh", self.doc) + def test_doc_lists_every_running_container_in_current_compose(self): - """The architecture table must reflect what ``docker compose up`` - actually produces today.""" missing = sorted(name for name in EXPECTED_CONTAINER_NAMES if name not in self.doc) self.assertFalse( missing, @@ -77,152 +78,94 @@ class MigrationDocumentationFactsTests(unittest.TestCase): ) def test_doc_warns_about_renamed_telegram_webhook_secret(self): - # This is the single rename most likely to bite a v2.7.0 → HEAD user. self.assertIn("TELEGRAM_WEBHOOK_SECRET", self.doc) self.assertIn("WEBHOOK_SECRET_TOKEN", self.doc) def test_doc_says_webhook_base_url_is_required(self): - # Polling mode was dropped — without WEBHOOK_BASE_URL the bot refuses - # to start. A user migrating from v2.7.0 (where it was optional) must - # be told this explicitly. block = self.doc.lower() self.assertIn("webhook_base_url", block) - # "обязательна" is the marker text in the env-vars table. self.assertIn("обязательн", block) def test_doc_mentions_migrate_one_shot_service(self): - # The migrate sidecar is what makes schema migrations transparent on - # the second-stage upgrade. Don't bury it. - self.assertIn("migrate", self.doc) - # "one-shot" or "разовый" / "однораз" text variants accepted. normalized = self.doc.lower() + self.assertIn("migrate", normalized) self.assertTrue( "one-shot" in normalized or "однораз" in normalized, "migration doc must describe `migrate` as a one-shot service", ) def test_doc_mentions_postgres_host_compose_override_caveat(self): - # Otherwise users follow the sed-fix step blindly and then panic - # because their .env still has the "wrong" hostname under compose. self.assertIn("POSTGRES_HOST", self.doc) - # Russian: "переопределя…" / "перебивает" indicate the override is documented. text = self.doc.lower() self.assertTrue( - "переопредел" in text or "перебивает" in text, + "переопредел" in text or "service name" in text, "migration doc must explain that compose overrides POSTGRES_HOST", ) def test_doc_describes_redis_data_and_shop_data_as_fresh(self): - # We do not migrate redis-data or shop-data — make sure that's said - # so users don't try to copy them from the old stack. text = self.doc.lower() self.assertIn("redis-data", text) self.assertIn("shop-data", text) - # Some phrasing variant must say it's empty / fresh / new on purpose. - self.assertTrue( - any(marker in text for marker in ("пустым", "создаётся пуст", "новые,", "пуст —")) - ) + self.assertTrue(any(marker in text for marker in ("пустым", "создается пуст"))) def test_doc_explains_reverse_proxy_no_longer_single_upstream(self): - # The note "rename remnawave-tg-shop → remnawave-minishop in your - # proxy config" used to be enough; after the split it's wrong. - # Make sure both backend:8080 and frontend:80 are documented. self.assertIn("backend:8080", self.doc) self.assertIn("frontend:80", self.doc) - -def _known_containers_from_script() -> set[str]: - """Parse the literal ``KNOWN_CONTAINERS`` array from the script source. - - The array is defined with ``${OLD_PREFIX}`` / ``${NEW_PREFIX}`` placeholders - that we substitute here. Doing this statically (rather than sourcing the - script in bash) avoids running ``main`` and keeps the test independent of - a bash interpreter being available at runtime. - """ - text = _read(SCRIPT_PATH) - - prefix_match = re.search(r'^OLD_PREFIX="([^"]+)"', text, flags=re.MULTILINE) - new_match = re.search(r'^NEW_PREFIX="([^"]+)"', text, flags=re.MULTILINE) - if not prefix_match or not new_match: - return set() - old_prefix = prefix_match.group(1) - new_prefix = new_match.group(1) - - # ``\n)`` as a boundary: a non-greedy ``.*?\)`` would stop at the first - # close-paren in a comment like ``# (v2.7.0 upstream remnawave-tg-shop)``. - array_match = re.search( - r"KNOWN_CONTAINERS=\((.*?)\n\)", - text, - flags=re.DOTALL, - ) - if not array_match: - return set() - - body = array_match.group(1) - # Strip line comments and quotes, then expand the two placeholders. - names: set[str] = set() - for raw_line in body.splitlines(): - line = raw_line.split("#", 1)[0].strip() - if not line: - continue - for token in re.findall(r'"([^"]+)"', line): - expanded = token.replace("${OLD_PREFIX}", old_prefix).replace( - "${NEW_PREFIX}", new_prefix - ) - names.add(expanded) - return names + def test_doc_mentions_both_supported_tgshop_migration_methods(self): + self.assertIn("Copy old Docker volumes", self.doc) + self.assertIn("Dump from a source PostgreSQL DSN", self.doc) + self.assertIn("pg_dump", self.doc) -class MigrationScriptCoverageTests(unittest.TestCase): +class InstallWizardCoverageTests(unittest.TestCase): def setUp(self) -> None: - self.script = _read(SCRIPT_PATH) - self.known = _known_containers_from_script() + self.script = _read(INSTALL_SCRIPT_PATH) + self.known = _known_containers_from_install_script() def test_known_containers_covers_split_arch(self): - """A re-run on a partially migrated stack must be able to stop the new - containers, otherwise ``docker compose up`` later fails with name - conflicts.""" missing = sorted(EXPECTED_CONTAINER_NAMES - self.known) self.assertFalse( missing, - f"KNOWN_CONTAINERS missing split-arch entries: {missing}\nactual: {sorted(self.known)}", + f"KNOWN_LEGACY_CONTAINERS missing split-arch entries: {missing}", ) - def test_known_containers_still_covers_legacy_eras(self): - # We must also keep stopping the original (v2.7.0) and intermediate - # (v3.1.x – v3.2.x) container names. + def test_known_containers_still_covers_old_eras(self): for legacy in ("remnawave-tg-shop", "remnawave-tg-shop-db", "remnawave-minishop-db"): with self.subTest(container=legacy): self.assertIn(legacy, self.known) - def test_script_is_syntactically_valid_bash(self): - # The script is curl|bash'ed from raw.githubusercontent in the docs, - # so a syntax break is a hard regression. - import shutil - import subprocess + def test_installer_contains_tgshop_volume_and_dsn_paths(self): + self.assertIn("run_tgshop_volume_migration", self.script) + self.assertIn("run_tgshop_dsn_migration", self.script) + self.assertIn("remnawave-tg-shop-db-data", self.script) + self.assertIn("pg_dump --clean --if-exists", self.script) + self.assertIn("run_compose run --rm migrate", self.script) - bash = shutil.which("bash") - if not bash: # pragma: no cover - self.skipTest("bash not available in PATH") + def test_script_is_syntactically_valid_sh_and_bash(self): + sh = shutil.which("sh") + if not sh: # pragma: no cover + self.skipTest("sh not available in PATH") result = subprocess.run( - [bash, "-n", str(SCRIPT_PATH)], + [sh, "-n", str(INSTALL_SCRIPT_PATH)], check=False, capture_output=True, text=True, ) - self.assertEqual( - result.returncode, - 0, - ( - "bash -n flagged migrate_to_minishop.sh:\n" - f"stdout={result.stdout}\nstderr={result.stderr}" - ), - ) + self.assertEqual(result.returncode, 0, result.stderr) + + bash = shutil.which("bash") + if bash: + result = subprocess.run( + [bash, "-n", str(INSTALL_SCRIPT_PATH)], + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) class DocComposeFileReferencesTests(unittest.TestCase): - """The doc links the user to specific compose files — they must exist.""" - def test_referenced_compose_files_exist(self): doc = _read(DOC_PATH) for relpath in ( @@ -234,40 +177,22 @@ class DocComposeFileReferencesTests(unittest.TestCase): ): with self.subTest(path=relpath): self.assertIn(relpath, doc) - self.assertTrue( - (REPO_ROOT / relpath).is_file(), - f"{relpath} is referenced in migrations/remnawave-tg-shop.md " - "but missing on disk", - ) + self.assertTrue((REPO_ROOT / relpath).is_file()) def test_doc_references_migrator_module_path(self): - # The doc tells users to expect ``backend/db/migrator.py`` migrations - # to apply via the migrate service. If the file moves, the doc lies. doc = _read(DOC_PATH) self.assertIn("backend/db/migrator.py", doc) self.assertTrue((REPO_ROOT / "backend" / "db" / "migrator.py").is_file()) class MigrationFootprintRegexTests(unittest.TestCase): - """Spot-check that what compose actually defines matches what we documented.""" - def test_every_compose_volume_documented(self): - """If a future compose file introduces a new ``remnawave-minishop-*`` - named volume, the migration doc must call out whether it carries data - from the old stack or starts fresh.""" doc = _read(DOC_PATH) compose_text = _all_compose_text() - # Find every named volume of the form ``remnawave-minishop--data``. defined = set(re.findall(r"remnawave-minishop-[\w-]+-data", compose_text)) - # Caddy-only volumes only ship in the caddy compose file but are still - # documented; allow them either way. for volume in defined: with self.subTest(volume=volume): - self.assertIn( - volume, - doc, - f"new volume {volume} is defined in compose but missing from migration doc", - ) + self.assertIn(volume, doc) if __name__ == "__main__": # pragma: no cover From 9508eece5f5e3cacf8b1c8cb1372139fd0f59357 Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 11:21:26 +0300 Subject: [PATCH 07/11] feat: import Remnashop payment settings --- backend/scripts/import_legacy.py | 596 ++++++++++++++++++++++++++- docs/getting-started/deployment.md | 6 +- docs/migrations/index.md | 2 +- docs/migrations/remnashop.md | 54 ++- scripts/install.sh | 64 ++- tests/test_install_script.py | 18 + tests/test_migration_doc_accuracy.py | 26 ++ tests/test_remnashop_import.py | 168 ++++++++ 8 files changed, 926 insertions(+), 8 deletions(-) diff --git a/backend/scripts/import_legacy.py b/backend/scripts/import_legacy.py index b2fede7..df3b0ed 100644 --- a/backend/scripts/import_legacy.py +++ b/backend/scripts/import_legacy.py @@ -17,6 +17,7 @@ import asyncio import json import logging import re +import shlex import sys from collections import defaultdict from datetime import datetime, timedelta, timezone @@ -53,13 +54,47 @@ from db.models import ( # noqa: E402 User, ) +try: # cryptography is already used by the app for payment webhook validation. + from cryptography.fernet import Fernet +except Exception: # pragma: no cover - defensive fallback for minimal tooling. + Fernet = None # type: ignore[assignment] + SOURCE = "remnashop" +REMNASHOP_ENCRYPTED_PREFIX = "enc_" GIB = 1024**3 UUID_RE = re.compile( r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" ) SAFE_SCHEMA_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +REMNASHOP_PAYMENT_WEBHOOK_PATH = "/api/v1/payments/{gateway}" +REMNASHOP_PANEL_WEBHOOK_PATH = "/api/v1/remnawave" + +SUPPORTED_REMNASHOP_PROVIDER_TYPES = { + "TELEGRAM_STARS", + "YOOKASSA", + "HELEKET", + "CRYPTOPAY", + "FREEKASSA", + "PLATEGA", + "WATA", +} +UNSUPPORTED_REMNASHOP_PROVIDER_TYPES = { + "YOOMONEY", + "CRYPTOMUS", + "MULENPAY", + "PAYMASTER", + "ROBOKASSA", + "URLPAY", +} +PAYMENT_WEBHOOK_PATHS = { + "yookassa": "/webhook/yookassa", + "wata": "/webhook/wata", + "cryptopay": "/webhook/cryptopay", + "heleket": "/webhook/heleket", + "freekassa": "/webhook/freekassa", + "platega": "/webhook/platega", +} logger = logging.getLogger(__name__) @@ -160,6 +195,365 @@ def _jsonish(value: Any) -> dict[str, Any]: return {} +def _strip_env_value(value: str) -> str: + lexer = shlex.shlex(value, posix=True) + lexer.whitespace_split = True + lexer.commenters = "#" + try: + tokens = list(lexer) + except ValueError: + return value.strip().strip("\"'") + return " ".join(tokens).strip() + + +def parse_remnashop_env_text(text_value: str) -> dict[str, str]: + env: dict[str, str] = {} + for raw_line in str(text_value or "").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[len("export ") :].strip() + if "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + if not key or not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key): + continue + env[key] = _strip_env_value(value) + return env + + +def read_remnashop_env_file(path: Optional[str]) -> dict[str, str]: + if not path: + return {} + return parse_remnashop_env_text(Path(path).read_text(encoding="utf-8")) + + +def _clean_url(value: Any) -> Optional[str]: + text_value = str(value or "").strip().rstrip("/") + return text_value or None + + +def _remnashop_panel_api_url(value: Any) -> Optional[str]: + host = _clean_url(value) + if not host: + return None + if "://" not in host: + if "." in host: + host = f"https://{host}" + else: + host = f"http://{host}:3000" + if not host.rstrip("/").endswith("/api"): + host = f"{host.rstrip('/')}/api" + return host + + +def _source_public_base_from_env(env: dict[str, str]) -> Optional[str]: + domain = _clean_url(env.get("APP_DOMAIN")) + if not domain: + return None + if "://" not in domain: + domain = f"https://{domain}" + return domain + + +def _support_link_from_username(value: Any) -> Optional[str]: + username = str(value or "").strip().lstrip("@") + if not username: + return None + return f"https://t.me/{username}" + + +def _mini_app_url_from_env(value: Any) -> Optional[str]: + raw = str(value or "").strip() + if not raw: + return None + if raw.lower() in {"true", "false", "0", "1"}: + return None + return raw if raw.startswith("https://") else None + + +def _add_override(overrides: dict[str, Any], key: str, value: Any) -> None: + if value is None: + return + if isinstance(value, str) and not value.strip(): + return + overrides[key] = value + + +def remnashop_env_overrides(env: dict[str, str]) -> dict[str, Any]: + overrides: dict[str, Any] = {} + _add_override(overrides, "PANEL_API_URL", _remnashop_panel_api_url(env.get("REMNAWAVE_HOST"))) + _add_override(overrides, "PANEL_API_KEY", env.get("REMNAWAVE_TOKEN")) + _add_override(overrides, "PANEL_WEBHOOK_SECRET", env.get("REMNAWAVE_WEBHOOK_SECRET")) + _add_override( + overrides, + "SUPPORT_LINK", + _support_link_from_username(env.get("BOT_SUPPORT_USERNAME")), + ) + _add_override(overrides, "DEFAULT_LANGUAGE", env.get("APP_DEFAULT_LOCALE")) + _add_override( + overrides, + "SUBSCRIPTION_MINI_APP_URL", + _mini_app_url_from_env(env.get("BOT_MINI_APP")), + ) + return overrides + + +def remnashop_source_urls_from_env(env: dict[str, str]) -> dict[str, str]: + base = _source_public_base_from_env(env) + if not base: + return {} + return { + "telegram": f"{base}/api/v1/telegram", + "remnawave_panel": f"{base}{REMNASHOP_PANEL_WEBHOOK_PATH}", + "payments": f"{base}/api/v1/payments/", + } + + +def _normalize_gateway_type(value: Any) -> str: + if hasattr(value, "value"): + value = value.value + text_value = str(value or "").strip().upper() + if "." in text_value: + text_value = text_value.rsplit(".", 1)[-1] + return re.sub(r"[^A-Z0-9_]+", "_", text_value).strip("_") + + +def _normalize_currency(value: Any) -> Optional[str]: + text_value = str(value or "").strip().upper() + if "." in text_value: + text_value = text_value.rsplit(".", 1)[-1] + aliases = {"RUR": "RUB", "STARS": "XTR", "STAR": "XTR"} + normalized = aliases.get(text_value, text_value) + return normalized or None + + +def _truthy(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return bool(value) + return str(value or "").strip().lower() in {"1", "true", "yes", "on", "active"} + + +def _is_encrypted_remnashop_value(value: Any) -> bool: + return isinstance(value, str) and value.startswith(REMNASHOP_ENCRYPTED_PREFIX) + + +def remnashop_decrypt_value(value: Any, crypt_key: Optional[str]) -> tuple[Any, bool]: + if not _is_encrypted_remnashop_value(value): + return value, False + if not crypt_key or Fernet is None: + return None, True + try: + token = str(value).removeprefix(REMNASHOP_ENCRYPTED_PREFIX).encode() + return Fernet(crypt_key.encode()).decrypt(token).decode(), False + except Exception: + return None, True + + +def remnashop_decrypt_recursive( + value: Any, + crypt_key: Optional[str], + *, + skipped_paths: Optional[list[str]] = None, + path: str = "", +) -> Any: + if isinstance(value, dict): + return { + key: remnashop_decrypt_recursive( + item, + crypt_key, + skipped_paths=skipped_paths, + path=f"{path}.{key}" if path else str(key), + ) + for key, item in value.items() + } + if isinstance(value, list): + return [ + remnashop_decrypt_recursive( + item, + crypt_key, + skipped_paths=skipped_paths, + path=f"{path}[{index}]", + ) + for index, item in enumerate(value) + ] + decrypted, skipped = remnashop_decrypt_value(value, crypt_key) + if skipped and skipped_paths is not None: + skipped_paths.append(path or "") + return decrypted + + +def _provider_mapping_result( + gateway_type: str, + provider_ids: Iterable[str], + overrides: dict[str, Any], + warnings: Optional[list[str]] = None, +) -> dict[str, Any]: + return { + "source_type": gateway_type, + "provider_ids": [provider for provider in provider_ids if provider], + "overrides": overrides, + "warnings": warnings or [], + "supported": True, + } + + +def remnashop_payment_gateway_overrides( + row: dict[str, Any], + *, + crypt_key: Optional[str] = None, +) -> dict[str, Any]: + gateway_type = _normalize_gateway_type(row.get("type")) + if gateway_type not in SUPPORTED_REMNASHOP_PROVIDER_TYPES: + return { + "source_type": gateway_type, + "provider_ids": [], + "overrides": {}, + "warnings": [], + "supported": False, + } + + skipped_secret_paths: list[str] = [] + settings = remnashop_decrypt_recursive( + _jsonish(row.get("settings")), + crypt_key, + skipped_paths=skipped_secret_paths, + ) + active = _truthy(row.get("is_active")) + currency = _normalize_currency(row.get("currency")) + overrides: dict[str, Any] = {} + warnings = [ + ( + f"Skipped encrypted Remnashop {gateway_type} setting '{path}': " + "APP_CRYPT_KEY is missing or invalid" + ) + for path in skipped_secret_paths + ] + + if gateway_type == "TELEGRAM_STARS": + _add_override(overrides, "STARS_ENABLED", active) + return _provider_mapping_result(gateway_type, ["stars"], overrides, warnings) + + if gateway_type == "YOOKASSA": + _add_override(overrides, "YOOKASSA_ENABLED", active) + _add_override(overrides, "YOOKASSA_SHOP_ID", settings.get("shop_id")) + _add_override(overrides, "YOOKASSA_SECRET_KEY", settings.get("api_key")) + _add_override(overrides, "YOOKASSA_DEFAULT_RECEIPT_EMAIL", settings.get("customer")) + _add_override(overrides, "YOOKASSA_VAT_CODE", settings.get("vat_code")) + if currency and currency != "RUB": + warnings.append( + f"YooKassa supports RUB only in this shop; source currency was {currency}" + ) + return _provider_mapping_result(gateway_type, ["yookassa"], overrides, warnings) + + if gateway_type == "WATA": + _add_override(overrides, "WATA_ENABLED", active) + _add_override(overrides, "WATA_API_TOKEN", settings.get("api_key")) + return _provider_mapping_result(gateway_type, ["wata"], overrides, warnings) + + if gateway_type == "CRYPTOPAY": + _add_override(overrides, "CRYPTOPAY_ENABLED", active) + _add_override(overrides, "CRYPTOPAY_TOKEN", settings.get("api_key")) + if currency: + _add_override(overrides, "CRYPTOPAY_ASSET", currency) + return _provider_mapping_result(gateway_type, ["cryptopay"], overrides, warnings) + + if gateway_type == "HELEKET": + _add_override(overrides, "HELEKET_ENABLED", active) + _add_override(overrides, "HELEKET_MERCHANT_ID", settings.get("merchant_id")) + _add_override(overrides, "HELEKET_API_KEY", settings.get("api_key")) + if currency: + _add_override(overrides, "HELEKET_CURRENCY", currency) + _add_override(overrides, "HELEKET_SUPPORTED_CURRENCIES", currency) + return _provider_mapping_result(gateway_type, ["heleket"], overrides, warnings) + + if gateway_type == "FREEKASSA": + _add_override(overrides, "FREEKASSA_ENABLED", active) + _add_override(overrides, "FREEKASSA_MERCHANT_ID", settings.get("shop_id")) + _add_override(overrides, "FREEKASSA_API_KEY", settings.get("api_key")) + _add_override(overrides, "FREEKASSA_SECOND_SECRET", settings.get("secret_word_2")) + _add_override(overrides, "FREEKASSA_PAYMENT_METHOD_ID", settings.get("payment_system_id")) + _add_override(overrides, "FREEKASSA_PAYMENT_IP", settings.get("customer_ip")) + if settings.get("customer_email"): + warnings.append( + "FreeKassa customer_email was captured by Remnashop but is not a " + "Minishop provider setting" + ) + return _provider_mapping_result(gateway_type, ["freekassa"], overrides, warnings) + + if gateway_type == "PLATEGA": + _add_override(overrides, "PLATEGA_ENABLED", active) + _add_override(overrides, "PLATEGA_SBP_ENABLED", active) + _add_override(overrides, "PLATEGA_MERCHANT_ID", settings.get("merchant_id")) + _add_override(overrides, "PLATEGA_SECRET", settings.get("api_key")) + _add_override(overrides, "PLATEGA_PAYMENT_METHOD", settings.get("payment_method")) + _add_override(overrides, "PLATEGA_SBP_METHOD", settings.get("payment_method")) + if currency: + _add_override(overrides, "PLATEGA_SUPPORTED_CURRENCIES", currency) + return _provider_mapping_result(gateway_type, ["platega_sbp"], overrides, warnings) + + return _provider_mapping_result(gateway_type, [], overrides, warnings) + + +def _target_webhook_url(base_url: Optional[str], path: str) -> Optional[str]: + base = _clean_url(base_url) + if not base: + return None + return f"{base}{path if path.startswith('/') else '/' + path}" + + +def remnashop_post_migration_actions( + *, + target_webhook_base_url: Optional[str], + imported_provider_ids: Iterable[str], + source_env: Optional[dict[str, str]] = None, +) -> dict[str, Any]: + provider_ids = list(dict.fromkeys(imported_provider_ids)) + payment_actions = [] + seen_paths: set[str] = set() + for provider_id in provider_ids: + path = PAYMENT_WEBHOOK_PATHS.get(provider_id) + if not path or path in seen_paths: + continue + seen_paths.add(path) + payment_actions.append( + { + "provider": provider_id, + "new_url": _target_webhook_url(target_webhook_base_url, path), + "where": { + "yookassa": "YooKassa merchant cabinet -> HTTP notifications URL", + "wata": "WATA merchant dashboard -> webhook/callback URL", + "cryptopay": "CryptoBot/Crypto Pay app -> webhook URL", + "heleket": "Heleket merchant dashboard -> payment webhook/callback URL", + "freekassa": "FreeKassa shop settings -> notification/result URL", + "platega": "Platega merchant/project settings -> webhook URL", + }.get(provider_id, "Payment provider dashboard -> webhook/callback URL"), + } + ) + + return { + "webhook_base_url_configured": bool(_clean_url(target_webhook_base_url)), + "source_urls": remnashop_source_urls_from_env(source_env or {}), + "remnawave_panel": { + "new_url": _target_webhook_url(target_webhook_base_url, "/webhook/panel"), + "where": "Remnawave Panel -> WEBHOOK_URL", + "secret": ( + "Set the Remnawave webhook secret to the value stored in " + "PANEL_WEBHOOK_SECRET." + ), + }, + "payment_providers": payment_actions, + "telegram": { + "new_url": _target_webhook_url(target_webhook_base_url, "/tg/webhook"), + "where": "Telegram webhook is set automatically by Minishop on startup.", + }, + } + + def _listish(value: Any) -> list[Any]: if value is None: return [] @@ -307,6 +701,9 @@ class RemnashopImporter: created_by_admin_id: int, tariff_map: dict[str, str], write_admin_compat_overrides: bool, + source_env: Optional[dict[str, str]] = None, + source_crypt_key: Optional[str] = None, + target_webhook_base_url: Optional[str] = None, ) -> None: self.source = source self.target = target @@ -317,8 +714,12 @@ class RemnashopImporter: self.created_by_admin_id = created_by_admin_id self.tariff_map = tariff_map self.write_admin_compat_overrides = write_admin_compat_overrides + self.source_env = source_env or {} + self.source_crypt_key = source_crypt_key or self.source_env.get("APP_CRYPT_KEY") + self.target_webhook_base_url = target_webhook_base_url self.tables: set[str] = set() self.user_map: dict[int, int] = {} + self.imported_payment_provider_ids: list[str] = [] self.summary: dict[str, Any] = { "source": SOURCE, "dry_run": dry_run, @@ -328,6 +729,7 @@ class RemnashopImporter: "subscriptions": _counter(), "payments": _counter(), "promocodes": _counter(), + "payment_provider_settings": _counter(), "settings": _counter(), "warnings": [], } @@ -349,6 +751,12 @@ class RemnashopImporter: if self._should_run("settings"): await self.import_settings() + self.summary["post_migration_actions"] = remnashop_post_migration_actions( + target_webhook_base_url=self.target_webhook_base_url, + imported_provider_ids=self.imported_payment_provider_ids, + source_env=self.source_env, + ) + if self.write_admin_compat_overrides: await self._write_admin_overrides() @@ -486,7 +894,19 @@ class RemnashopImporter: result = await self.target.execute(stmt) return result.scalar_one_or_none() - async def _upsert_setting_override(self, key: str, value: Any) -> None: + async def _upsert_setting_override(self, key: str, value: Any) -> bool: + from bot.app.web.admin_settings_manifest import coerce_value, get_field_by_key + + field = get_field_by_key(key) + if field is None: + self.summary["warnings"].append(f"Skipped unknown admin setting override: {key}") + return False + try: + value = coerce_value(field, value) + except ValueError as exc: + self.summary["warnings"].append(f"Skipped invalid admin setting override {key}: {exc}") + return False + now = datetime.now(timezone.utc) encoded = json.dumps(value, ensure_ascii=False, separators=(",", ":")) stmt = ( @@ -507,6 +927,142 @@ class RemnashopImporter: ) ) await self.target.execute(stmt) + return True + + async def _write_setting_overrides( + self, + overrides: dict[str, Any], + *, + summary_key: str, + ) -> list[str]: + written: list[str] = [] + for key, value in overrides.items(): + if await self._upsert_setting_override(key, value): + written.append(key) + self.summary[summary_key]["overrides_written"] += 1 + else: + self.summary[summary_key]["overrides_skipped"] += 1 + return written + + async def import_env_settings(self) -> list[str]: + if not self.source_env: + self.summary["settings"]["source_env_missing"] += 1 + return [] + + overrides = remnashop_env_overrides(self.source_env) + if not overrides: + self.summary["settings"]["source_env_no_supported_values"] += 1 + return [] + + written = await self._write_setting_overrides(overrides, summary_key="settings") + if written: + self.summary["settings"]["source_env_overrides_written"] += 1 + await self._upsert_mapping( + entity_type="settings_env", + source_id="remnashop.env", + target_table="app_setting_overrides", + target_id=",".join(written) if written else "none", + metadata={ + "override_keys": written, + "source_keys_used": sorted( + key + for key in ( + "REMNAWAVE_HOST", + "REMNAWAVE_TOKEN", + "REMNAWAVE_WEBHOOK_SECRET", + "BOT_SUPPORT_USERNAME", + "APP_DEFAULT_LOCALE", + "BOT_MINI_APP", + ) + if self.source_env.get(key) + ), + "has_app_crypt_key": bool(self.source_env.get("APP_CRYPT_KEY")), + "source_urls": remnashop_source_urls_from_env(self.source_env), + }, + ) + return written + + async def import_payment_provider_settings(self) -> None: + if "payment_gateways" not in self.tables: + self.summary["payment_provider_settings"]["missing_source_table"] += 1 + return + + rows = await self._fetch_rows("payment_gateways", order_by="order_index, id") + if not rows: + self.summary["payment_provider_settings"]["empty_source_table"] += 1 + return + + active_provider_ids: list[str] = [] + for index, row in enumerate(rows): + source_id = row.get("id") or row.get("type") or f"row:{index}" + mapping = remnashop_payment_gateway_overrides( + row, + crypt_key=self.source_crypt_key, + ) + gateway_type = mapping["source_type"] + self.summary["payment_provider_settings"]["seen"] += 1 + + for warning in mapping["warnings"]: + self.summary["warnings"].append(warning) + + if not mapping["supported"]: + self.summary["payment_provider_settings"]["unsupported"] += 1 + display_type = gateway_type or str(row.get("type") or "unknown") + self.summary["warnings"].append( + f"Remnashop payment provider {display_type} is not supported by " + "Minishop; configure it manually if it is still needed." + ) + await self._upsert_mapping( + entity_type="payment_provider_settings", + source_id=source_id, + target_table="manual_configuration_required", + target_id=display_type, + metadata={ + "source_type": display_type, + "active": _truthy(row.get("is_active")), + "currency": _normalize_currency(row.get("currency")), + "supported": False, + }, + ) + continue + + written = await self._write_setting_overrides( + mapping["overrides"], + summary_key="payment_provider_settings", + ) + if written: + self.summary["payment_provider_settings"]["providers_mapped"] += 1 + else: + self.summary["payment_provider_settings"]["providers_without_overrides"] += 1 + + if _truthy(row.get("is_active")): + for provider_id in mapping["provider_ids"]: + if provider_id and provider_id not in active_provider_ids: + active_provider_ids.append(provider_id) + if provider_id and provider_id not in self.imported_payment_provider_ids: + self.imported_payment_provider_ids.append(provider_id) + + await self._upsert_mapping( + entity_type="payment_provider_settings", + source_id=source_id, + target_table="app_setting_overrides", + target_id=",".join(written) if written else "none", + metadata={ + "source_type": gateway_type, + "provider_ids": mapping["provider_ids"], + "active": _truthy(row.get("is_active")), + "currency": _normalize_currency(row.get("currency")), + "override_keys": written, + "source_settings_keys": sorted(_jsonish(row.get("settings")).keys()), + "warnings_count": len(mapping["warnings"]), + "supported": True, + }, + ) + + if active_provider_ids: + order_value = ",".join(active_provider_ids) + if await self._upsert_setting_override("PAYMENT_METHODS_ORDER", order_value): + self.summary["payment_provider_settings"]["payment_order_written"] += 1 async def _upsert_legacy_referral_code(self, *, code: str, user_id: int) -> None: if len(code) > 128: @@ -1029,7 +1585,29 @@ class RemnashopImporter: } for plan in plans[:100] ], + "source_env": { + "provided": bool(self.source_env), + "supported_keys_present": sorted( + key + for key in ( + "REMNAWAVE_HOST", + "REMNAWAVE_TOKEN", + "REMNAWAVE_WEBHOOK_SECRET", + "BOT_SUPPORT_USERNAME", + "APP_DEFAULT_LOCALE", + "BOT_MINI_APP", + "APP_DOMAIN", + "APP_CRYPT_KEY", + ) + if self.source_env.get(key) + ), + "source_urls": remnashop_source_urls_from_env(self.source_env), + }, } + env_override_keys = await self.import_env_settings() + await self.import_payment_provider_settings() + notes["env_override_keys"] = env_override_keys + notes["payment_provider_ids"] = list(dict.fromkeys(self.imported_payment_provider_ids)) await self._upsert_mapping( entity_type="settings", source_id="singleton", @@ -1080,6 +1658,17 @@ def build_arg_parser() -> argparse.ArgumentParser: parser.add_argument("--source-type", choices=[SOURCE], default=SOURCE) parser.add_argument("--source-dsn", required=True) parser.add_argument("--source-schema", default="public") + parser.add_argument( + "--source-env-file", + help=( + "Path to the source Remnashop .env. Used for APP_CRYPT_KEY, Remnawave " + "API settings and selected safe compatibility values." + ), + ) + parser.add_argument( + "--source-crypt-key", + help="Explicit Remnashop APP_CRYPT_KEY. Overrides the value from --source-env-file.", + ) parser.add_argument("--target-dsn") parser.add_argument( "--only", @@ -1116,6 +1705,8 @@ async def _prepare_target_schema(engine: Any) -> None: async def run_import(args: argparse.Namespace) -> dict[str, Any]: settings = Settings() + source_env = read_remnashop_env_file(args.source_env_file) + source_crypt_key = args.source_crypt_key or source_env.get("APP_CRYPT_KEY") source_engine = create_async_engine(normalize_async_postgres_dsn(args.source_dsn)) target_engine = create_async_engine( normalize_async_postgres_dsn(args.target_dsn or settings.DATABASE_URL) @@ -1141,6 +1732,9 @@ async def run_import(args: argparse.Namespace) -> dict[str, Any]: created_by_admin_id=args.created_by_admin_id, tariff_map=parse_tariff_map(args.tariff_map_json), write_admin_compat_overrides=not args.no_admin_compat_overrides, + source_env=source_env, + source_crypt_key=source_crypt_key, + target_webhook_base_url=settings.WEBHOOK_BASE_URL, ) summary = await importer.run() if args.dry_run: diff --git a/docs/getting-started/deployment.md b/docs/getting-started/deployment.md index 1df27b0..e5ff0bb 100644 --- a/docs/getting-started/deployment.md +++ b/docs/getting-started/deployment.md @@ -42,7 +42,11 @@ sh install.sh ``` Миграция Remnashop в wizard сначала запускает `dry-run`, показывает JSON-сводку -и только после отдельного подтверждения применяет изменения в целевую БД. +и только после отдельного подтверждения применяет изменения в целевую БД. Если +указать старый Remnashop `.env`, wizard передаст importer-у `APP_CRYPT_KEY`, +Remnawave API settings и поддерживаемые payment provider settings из таблицы +`payment_gateways`. После применения wizard печатает новые webhook URL для +Remnawave Panel и платежных провайдеров. Миграция со старого `remnawave-tg-shop` работает как upgrade совместимой БД: либо копирует старый Docker volume, либо делает `pg_dump` по source DSN, восстанавливает дамп в целевую compose-БД и запускает сервис `migrate`. diff --git a/docs/migrations/index.md b/docs/migrations/index.md index 0c20758..42d2a74 100644 --- a/docs/migrations/index.md +++ b/docs/migrations/index.md @@ -5,4 +5,4 @@ | Источник | Поддерживаемый случай | Документы | | --- | --- | --- | | [remnawave-tg-shop](https://github.com/kavore/remnawave-tg-shop/) | Полный перенос всех данных | [Инструкция](remnawave-tg-shop.md) | -| [Remnashop](https://github.com/snoups/remnashop/) | Автоматический импорт пользователей, подписок, платежей, рефералов и промокодов | [Инструкция](remnashop.md) | +| [Remnashop](https://github.com/snoups/remnashop/) | Автоматический импорт пользователей, подписок, платежей, рефералов, промокодов и поддерживаемых платежных настроек | [Инструкция](remnashop.md) | diff --git a/docs/migrations/remnashop.md b/docs/migrations/remnashop.md index 11e662c..a719c4a 100644 --- a/docs/migrations/remnashop.md +++ b/docs/migrations/remnashop.md @@ -24,16 +24,61 @@ sh install.sh Данные, которые не имеют прямого аналога, сохраняются в служебных таблицах миграции или message logs как заметки, чтобы администратор мог проверить их после переноса. +## Настройки и платежные провайдеры + +Если указать старый Remnashop `.env`, importer дополнительно переносит часть +настроек в админские overrides: + +- `REMNAWAVE_HOST` -> `PANEL_API_URL`; +- `REMNAWAVE_TOKEN` -> `PANEL_API_KEY`; +- `REMNAWAVE_WEBHOOK_SECRET` -> `PANEL_WEBHOOK_SECRET`; +- `BOT_SUPPORT_USERNAME` -> `SUPPORT_LINK`; +- `APP_DEFAULT_LOCALE` -> `DEFAULT_LANGUAGE`; +- `BOT_MINI_APP` -> `SUBSCRIPTION_MINI_APP_URL`, если там уже HTTPS URL. + +Платежные провайдеры берутся из таблицы Remnashop `payment_gateways`. +Поддерживаются и автоматически маппятся: Telegram Stars, YooKassa, WATA, +CryptoPay, Heleket, FreeKassa и Platega. Для них importer переносит флаги +включения, API-ключи/merchant IDs и доступные provider-specific параметры в +раздел настроек админки. + +Провайдеры YooMoney, Cryptomus, MulenPay, PayMaster, RoboKassa и UrlPay сейчас +не имеют прямого аналога в Minishop. Если они были в Remnashop, importer +оставит предупреждение в JSON-сводке и notes миграции, а настроить их нужно +вручную или через будущий отдельный provider. + +Remnashop может хранить секреты в формате `enc_...`. Для расшифровки нужен +старый `APP_CRYPT_KEY`; проще всего указать путь к старому `.env` в wizard или +передать `--source-env-file`. Если ключ не передан или неверный, зашифрованные +значения будут пропущены с предупреждением, остальные данные продолжат +импортироваться. + +После успешного применения wizard печатает список новых адресов webhook. Их +нужно указать во внешних сервисах вместо старых Remnashop URL: + +- Remnawave Panel -> `WEBHOOK_URL`: `WEBHOOK_BASE_URL` + `/webhook/panel`; +- YooKassa HTTP notifications URL: `WEBHOOK_BASE_URL` + `/webhook/yookassa`; +- WATA webhook/callback URL: `WEBHOOK_BASE_URL` + `/webhook/wata`; +- CryptoBot/Crypto Pay webhook URL: `WEBHOOK_BASE_URL` + `/webhook/cryptopay`; +- Heleket payment webhook/callback URL: `WEBHOOK_BASE_URL` + `/webhook/heleket`; +- FreeKassa notification/result URL: `WEBHOOK_BASE_URL` + `/webhook/freekassa`; +- Platega webhook URL: `WEBHOOK_BASE_URL` + `/webhook/platega`; +- Telegram webhook `WEBHOOK_BASE_URL` + `/tg/webhook` выставляется ботом + автоматически при старте. + ## Flow wizard 1. Wizard скачивает compose-профиль и `backend/scripts/import_legacy.py` через `raw.githubusercontent.com`, без клонирования репозитория. 2. Вы указываете source PostgreSQL DSN Remnashop и schema, обычно `public`. -3. Вы выбираете целевую БД: текущую compose-БД или ручной target DSN. -4. При необходимости указываете JSON map тарифов Remnashop в локальные +3. Опционально указываете путь к старому Remnashop `.env` для `APP_CRYPT_KEY`, + Remnawave API settings и переносимых payment/provider settings. +4. Вы выбираете целевую БД: текущую compose-БД или ручной target DSN. +5. При необходимости указываете JSON map тарифов Remnashop в локальные `tariff_key`, например `{"basic": "standard_month"}`. -5. Wizard запускает `dry-run` и показывает JSON-сводку. -6. После подтверждения `y` importer применяет изменения и перезапускает +6. Wizard запускает `dry-run` и показывает JSON-сводку. +7. После подтверждения `y` importer применяет изменения, печатает список новых + webhook URL для Remnawave Panel и платежных провайдеров, затем перезапускает `backend`/`worker`, чтобы настройки совместимости перечитались. Если source DB находится на том же Docker host, помните, что DSN выполняется @@ -51,6 +96,7 @@ docker compose run --rm backend \ --source-type remnashop \ --source-dsn 'postgresql://old_user:old_password@old_host:5432/remnashop' \ --source-schema public \ + --source-env-file /path/to/remnashop/.env \ --dry-run ``` diff --git a/scripts/install.sh b/scripts/install.sh index 95b7835..b4b5caf 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -47,6 +47,7 @@ COMPOSE_STYLE="" PROMPT_VALUE="" CHOICE_VALUE="" LEGACY_SOURCE="" +SOURCE_ENV_PATH="" COMPOSE_PROJECT_NAME_VALUE="" IMAGE_TAG_VALUE="" @@ -128,6 +129,7 @@ Environment overrides: MINISHOP_INSTALL_REF default ref ($DEFAULT_REF) MINISHOP_IMAGE_TAG default image tag ($DEFAULT_IMAGE_TAG) REMNASHOP_SOURCE_DSN default source DSN for migration + REMNASHOP_SOURCE_ENV_FILE default source Remnashop .env path for migration LEGACY_TGSHOP_SOURCE_DSN default remnawave-tg-shop source DSN for dump/restore The wizard is interactive by design. It never overwrites files without @@ -838,10 +840,52 @@ local_target_dsn() { printf 'postgresql://%s:%s@postgres:5432/%s' "$POSTGRES_USER_VALUE" "$POSTGRES_PASSWORD_VALUE" "$POSTGRES_DB_VALUE" } +target_webhook_base_url() { + public_url=$(env_get WEBHOOK_PUBLIC_URL "") + if [ -n "$public_url" ]; then + printf '%s' "$public_url" | sed 's:/*$::' + return 0 + fi + host=$(env_get WEBHOOK_HOST "") + if [ -n "$host" ]; then + printf 'https://%s' "$host" | sed 's:/*$::' + return 0 + fi + printf '' +} + +remnashop_webhook_checklist() { + section "Update external webhooks" + base_url=$(target_webhook_base_url) + if [ -z "$base_url" ]; then + warn "Could not determine webhook base URL from .env. Set WEBHOOK_HOST or WEBHOOK_PUBLIC_URL, then use WEBHOOK_BASE_URL + paths below." + base_url="WEBHOOK_BASE_URL" + fi + + info "Set these URLs in external dashboards after the migration:" + printf ' Remnawave Panel -> WEBHOOK_URL: %s/webhook/panel\n' "$base_url" + panel_secret=$(env_get PANEL_WEBHOOK_SECRET "") + if [ -n "$panel_secret" ]; then + printf ' Remnawave Panel -> webhook secret: %s\n' "$(mask_secret "$panel_secret")" + else + warn "PANEL_WEBHOOK_SECRET is empty; set it in Minishop and in Remnawave Panel." + fi + printf ' YooKassa merchant cabinet -> HTTP notifications URL: %s/webhook/yookassa\n' "$base_url" + printf ' WATA merchant dashboard -> webhook/callback URL: %s/webhook/wata\n' "$base_url" + printf ' CryptoBot/Crypto Pay app -> webhook URL: %s/webhook/cryptopay\n' "$base_url" + printf ' Heleket merchant dashboard -> payment webhook/callback URL: %s/webhook/heleket\n' "$base_url" + printf ' FreeKassa shop settings -> notification/result URL: %s/webhook/freekassa\n' "$base_url" + printf ' Platega merchant/project settings -> webhook URL: %s/webhook/platega\n' "$base_url" + printf ' Telegram webhook: %s/tg/webhook (configured automatically on bot startup)\n' "$base_url" +} + run_import_command() { dry="$1" set -- run --rm \ -v "$IMPORTER_PATH:/app/backend/scripts/import_legacy.py:ro" + if [ -n "$SOURCE_ENV_PATH" ]; then + set -- "$@" -v "$SOURCE_ENV_PATH:/tmp/remnashop.env:ro" + fi if [ -n "$TARIFF_MAP_PATH" ]; then set -- "$@" -v "$TARIFF_MAP_PATH:/tmp/tariff-map.json:ro" fi @@ -850,6 +894,9 @@ run_import_command() { --source-dsn "$SOURCE_DSN" \ --source-schema "$SOURCE_SCHEMA" \ --target-dsn "$TARGET_DSN" + if [ -n "$SOURCE_ENV_PATH" ]; then + set -- "$@" --source-env-file /tmp/remnashop.env + fi if [ -n "$TARIFF_MAP_PATH" ]; then set -- "$@" --tariff-map-json /tmp/tariff-map.json fi @@ -861,7 +908,7 @@ run_import_command() { choose_legacy_source() { choose "Source bot" "1" "1|2|3" \ - "1. Remnashop - import users, subscriptions, payments, referrals and promo codes." \ + "1. Remnashop - import users, subscriptions, payments, provider settings and promo codes." \ "2. Old remnawave-tg-shop - upgrade an old compatible database/volume." \ "3. Skip migration" case "$CHOICE_VALUE" in @@ -895,6 +942,20 @@ run_remnashop_migration() { SOURCE_DSN="$PROMPT_VALUE" prompt_value "Source schema" "public" 1 0 "" SOURCE_SCHEMA="$PROMPT_VALUE" + prompt_value "Optional source Remnashop .env path (empty to skip)" "${REMNASHOP_SOURCE_ENV_FILE:-}" 0 0 "" + SOURCE_ENV_PATH="$PROMPT_VALUE" + if [ -n "$SOURCE_ENV_PATH" ]; then + source_env_dir=$(dirname "$SOURCE_ENV_PATH") + if [ ! -d "$source_env_dir" ]; then + fail "Source .env directory not found: $source_env_dir" + return 1 + fi + SOURCE_ENV_PATH=$(cd "$source_env_dir" && pwd)/$(basename "$SOURCE_ENV_PATH") + if [ ! -f "$SOURCE_ENV_PATH" ]; then + fail "Source Remnashop .env not found: $SOURCE_ENV_PATH" + return 1 + fi + fi choose "Target database" "1" "1|2" \ "1. This Docker Compose stack database (recommended)" \ @@ -936,6 +997,7 @@ run_remnashop_migration() { section "Apply import" run_import_command 0 || return 1 + remnashop_webhook_checklist if confirm "Restart backend and worker so setting overrides are reloaded?" 1; then (cd "$TARGET_DIR" && run_compose restart backend worker) || true fi diff --git a/tests/test_install_script.py b/tests/test_install_script.py index 654627e..bc2477c 100644 --- a/tests/test_install_script.py +++ b/tests/test_install_script.py @@ -36,6 +36,8 @@ def test_shell_installer_downloads_raw_files_and_runs_import_in_container(): assert "raw.githubusercontent.com" in script assert "git clone" not in script assert "backend python backend/scripts/import_legacy.py" in script + assert "Optional source Remnashop .env path" in script + assert "--source-env-file /tmp/remnashop.env" in script assert "--dry-run" in script assert "Install new stack and run migration" in script assert "Run migration only" in script @@ -61,3 +63,19 @@ def test_shell_installer_only_prepares_data_mount_not_runtime_content(): assert "webapp-logo" not in script assert "webapp-emoji" not in script assert "locales-overrides.json" not in script + + +def test_shell_installer_prints_remnashop_webhook_checklist(): + script = INSTALL_SCRIPT.read_text(encoding="utf-8") + + assert "remnashop_webhook_checklist" in script + assert "Remnawave Panel -> WEBHOOK_URL" in script + assert "PANEL_WEBHOOK_SECRET" in script + assert "/webhook/panel" in script + assert "/webhook/yookassa" in script + assert "/webhook/wata" in script + assert "/webhook/cryptopay" in script + assert "/webhook/heleket" in script + assert "/webhook/freekassa" in script + assert "/webhook/platega" in script + assert "/tg/webhook" in script diff --git a/tests/test_migration_doc_accuracy.py b/tests/test_migration_doc_accuracy.py index 5dd79b7..6c0b850 100644 --- a/tests/test_migration_doc_accuracy.py +++ b/tests/test_migration_doc_accuracy.py @@ -8,6 +8,7 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] DOC_PATH = REPO_ROOT / "docs" / "migrations" / "remnawave-tg-shop.md" +REMNASHOP_DOC_PATH = REPO_ROOT / "docs" / "migrations" / "remnashop.md" INSTALL_SCRIPT_PATH = REPO_ROOT / "scripts" / "install.sh" REMOVED_SCRIPT_PATH = REPO_ROOT / "scripts" / "migrate_to_minishop.sh" COMPOSE_FILES = ( @@ -185,6 +186,31 @@ class DocComposeFileReferencesTests(unittest.TestCase): self.assertTrue((REPO_ROOT / "backend" / "db" / "migrator.py").is_file()) +class RemnashopMigrationDocumentationFactsTests(unittest.TestCase): + def setUp(self) -> None: + self.doc = _read(REMNASHOP_DOC_PATH) + + def test_doc_mentions_env_payment_gateways_and_encrypted_secrets(self): + self.assertIn("--source-env-file", self.doc) + self.assertIn("APP_CRYPT_KEY", self.doc) + self.assertIn("payment_gateways", self.doc) + self.assertIn("enc_", self.doc) + + def test_doc_lists_new_webhook_paths_after_migration(self): + for path in ( + "/webhook/panel", + "/webhook/yookassa", + "/webhook/wata", + "/webhook/cryptopay", + "/webhook/heleket", + "/webhook/freekassa", + "/webhook/platega", + "/tg/webhook", + ): + with self.subTest(path=path): + self.assertIn(path, self.doc) + + class MigrationFootprintRegexTests(unittest.TestCase): def test_every_compose_volume_documented(self): doc = _read(DOC_PATH) diff --git a/tests/test_remnashop_import.py b/tests/test_remnashop_import.py index fb4a16b..0010ef1 100644 --- a/tests/test_remnashop_import.py +++ b/tests/test_remnashop_import.py @@ -1,7 +1,12 @@ from datetime import datetime, timezone +import pytest from scripts.import_legacy import ( + parse_remnashop_env_text, + remnashop_env_overrides, remnashop_months_from_plan_snapshot, + remnashop_payment_gateway_overrides, + remnashop_post_migration_actions, remnashop_pricing_amount, remnashop_pricing_currency, remnashop_sale_mode, @@ -42,3 +47,166 @@ def test_remnashop_plan_months_prefers_snapshot_then_dates(): ) == 3 ) + + +def test_remnashop_env_parser_and_overrides_map_safe_values(): + env = parse_remnashop_env_text( + """ + # old Remnashop + export REMNAWAVE_HOST=panel.example.com + REMNAWAVE_TOKEN='panel-token#kept' + REMNAWAVE_WEBHOOK_SECRET="panel secret" + BOT_SUPPORT_USERNAME=@support_bot # comment + APP_DEFAULT_LOCALE=en + BOT_MINI_APP=https://app.example.com/ + APP_DOMAIN=old.example.com + """ + ) + + assert env["REMNAWAVE_TOKEN"] == "panel-token#kept" + assert env["BOT_SUPPORT_USERNAME"] == "@support_bot" + + overrides = remnashop_env_overrides(env) + assert overrides == { + "PANEL_API_URL": "https://panel.example.com/api", + "PANEL_API_KEY": "panel-token#kept", + "PANEL_WEBHOOK_SECRET": "panel secret", + "SUPPORT_LINK": "https://t.me/support_bot", + "DEFAULT_LANGUAGE": "en", + "SUBSCRIPTION_MINI_APP_URL": "https://app.example.com/", + } + + +def test_remnashop_yookassa_gateway_maps_to_current_provider_settings(): + result = remnashop_payment_gateway_overrides( + { + "type": "YOOKASSA", + "currency": "RUB", + "is_active": True, + "settings": { + "shop_id": "shop-1", + "api_key": "secret", + "customer": "receipt@example.com", + "vat_code": 1, + }, + } + ) + + assert result["supported"] is True + assert result["provider_ids"] == ["yookassa"] + assert result["overrides"] == { + "YOOKASSA_ENABLED": True, + "YOOKASSA_SHOP_ID": "shop-1", + "YOOKASSA_SECRET_KEY": "secret", + "YOOKASSA_DEFAULT_RECEIPT_EMAIL": "receipt@example.com", + "YOOKASSA_VAT_CODE": 1, + } + + +def test_remnashop_free_kassa_and_platega_gateways_map_available_settings(): + freekassa = remnashop_payment_gateway_overrides( + { + "type": "FREEKASSA", + "is_active": True, + "settings": { + "shop_id": "merchant", + "api_key": "api", + "secret_word_2": "notify-secret", + "payment_system_id": 42, + "customer_ip": "203.0.113.10", + "customer_email": "payer@example.com", + }, + } + ) + assert freekassa["provider_ids"] == ["freekassa"] + assert freekassa["overrides"]["FREEKASSA_SECOND_SECRET"] == "notify-secret" + assert freekassa["overrides"]["FREEKASSA_PAYMENT_METHOD_ID"] == 42 + assert any("customer_email" in warning for warning in freekassa["warnings"]) + + platega = remnashop_payment_gateway_overrides( + { + "type": "PLATEGA", + "currency": "RUB", + "is_active": True, + "settings": { + "merchant_id": "merchant", + "api_key": "secret", + "payment_method": 2, + }, + } + ) + assert platega["provider_ids"] == ["platega_sbp"] + assert platega["overrides"]["PLATEGA_SBP_ENABLED"] is True + assert platega["overrides"]["PLATEGA_SBP_METHOD"] == 2 + + +def test_remnashop_unsupported_gateway_is_reported_without_overrides(): + result = remnashop_payment_gateway_overrides( + { + "type": "ROBOKASSA", + "is_active": True, + "settings": {"merchant_login": "shop"}, + } + ) + + assert result["supported"] is False + assert result["provider_ids"] == [] + assert result["overrides"] == {} + + +def test_remnashop_encrypted_gateway_settings_need_app_crypt_key(): + result = remnashop_payment_gateway_overrides( + { + "type": "WATA", + "is_active": True, + "settings": {"api_key": "enc_not-a-fernet-token"}, + } + ) + + assert result["overrides"] == {"WATA_ENABLED": True} + assert any("APP_CRYPT_KEY" in warning for warning in result["warnings"]) + + +def test_remnashop_encrypted_gateway_settings_decrypt_with_app_crypt_key(): + cryptography = pytest.importorskip("cryptography.fernet") + key = cryptography.Fernet.generate_key().decode() + token = cryptography.Fernet(key.encode()).encrypt(b"wata-token").decode() + + result = remnashop_payment_gateway_overrides( + { + "type": "WATA", + "is_active": True, + "settings": {"api_key": f"enc_{token}"}, + }, + crypt_key=key, + ) + + assert result["overrides"] == { + "WATA_ENABLED": True, + "WATA_API_TOKEN": "wata-token", + } + assert result["warnings"] == [] + + +def test_remnashop_post_migration_actions_include_new_webhook_urls(): + actions = remnashop_post_migration_actions( + target_webhook_base_url="https://webhooks.example.com/", + imported_provider_ids=["yookassa", "wata", "yookassa"], + source_env={"APP_DOMAIN": "old.example.com"}, + ) + + assert actions["remnawave_panel"]["new_url"] == "https://webhooks.example.com/webhook/panel" + assert actions["telegram"]["new_url"] == "https://webhooks.example.com/tg/webhook" + assert actions["source_urls"]["payments"] == "https://old.example.com/api/v1/payments/" + assert actions["payment_providers"] == [ + { + "provider": "yookassa", + "new_url": "https://webhooks.example.com/webhook/yookassa", + "where": "YooKassa merchant cabinet -> HTTP notifications URL", + }, + { + "provider": "wata", + "new_url": "https://webhooks.example.com/webhook/wata", + "where": "WATA merchant dashboard -> webhook/callback URL", + }, + ] From e9e028fc00873953d5c602cc521a572c9f53f785 Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 11:31:03 +0300 Subject: [PATCH 08/11] fix: narrow Remnashop settings import --- backend/scripts/import_legacy.py | 66 ++++++++++++++++++++------------ docs/migrations/remnashop.md | 26 ++++++++++--- tests/test_remnashop_import.py | 48 ++++++++++++++++++++++- 3 files changed, 109 insertions(+), 31 deletions(-) diff --git a/backend/scripts/import_legacy.py b/backend/scripts/import_legacy.py index df3b0ed..f3dbd0b 100644 --- a/backend/scripts/import_legacy.py +++ b/backend/scripts/import_legacy.py @@ -61,6 +61,7 @@ except Exception: # pragma: no cover - defensive fallback for minimal tooling. SOURCE = "remnashop" REMNASHOP_ENCRYPTED_PREFIX = "enc_" +PLACEHOLDER_SETTING_VALUES = {"change_me", "changeme"} GIB = 1024**3 UUID_RE = re.compile( r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" @@ -230,7 +231,13 @@ def read_remnashop_env_file(path: Optional[str]) -> dict[str, str]: return parse_remnashop_env_text(Path(path).read_text(encoding="utf-8")) +def _is_placeholder_setting_value(value: Any) -> bool: + return isinstance(value, str) and value.strip().lower() in PLACEHOLDER_SETTING_VALUES + + def _clean_url(value: Any) -> Optional[str]: + if _is_placeholder_setting_value(value): + return None text_value = str(value or "").strip().rstrip("/") return text_value or None @@ -259,26 +266,23 @@ def _source_public_base_from_env(env: dict[str, str]) -> Optional[str]: def _support_link_from_username(value: Any) -> Optional[str]: + if _is_placeholder_setting_value(value): + return None username = str(value or "").strip().lstrip("@") if not username: return None return f"https://t.me/{username}" -def _mini_app_url_from_env(value: Any) -> Optional[str]: - raw = str(value or "").strip() - if not raw: - return None - if raw.lower() in {"true", "false", "0", "1"}: - return None - return raw if raw.startswith("https://") else None - - def _add_override(overrides: dict[str, Any], key: str, value: Any) -> None: if value is None: return - if isinstance(value, str) and not value.strip(): - return + if isinstance(value, str): + value = value.strip() + if not value: + return + if value.lower() in PLACEHOLDER_SETTING_VALUES: + return overrides[key] = value @@ -293,11 +297,6 @@ def remnashop_env_overrides(env: dict[str, str]) -> dict[str, Any]: _support_link_from_username(env.get("BOT_SUPPORT_USERNAME")), ) _add_override(overrides, "DEFAULT_LANGUAGE", env.get("APP_DEFAULT_LOCALE")) - _add_override( - overrides, - "SUBSCRIPTION_MINI_APP_URL", - _mini_app_url_from_env(env.get("BOT_MINI_APP")), - ) return overrides @@ -458,17 +457,24 @@ def remnashop_payment_gateway_overrides( if gateway_type == "CRYPTOPAY": _add_override(overrides, "CRYPTOPAY_ENABLED", active) _add_override(overrides, "CRYPTOPAY_TOKEN", settings.get("api_key")) - if currency: - _add_override(overrides, "CRYPTOPAY_ASSET", currency) + if currency and currency != "RUB": + warnings.append( + f"CryptoPay source currency was {currency}; Minishop keeps payment currency " + "controlled by tariffs/default currency. Configure CRYPTOPAY_ASSET manually " + "if this instance needs a different default." + ) return _provider_mapping_result(gateway_type, ["cryptopay"], overrides, warnings) if gateway_type == "HELEKET": _add_override(overrides, "HELEKET_ENABLED", active) _add_override(overrides, "HELEKET_MERCHANT_ID", settings.get("merchant_id")) _add_override(overrides, "HELEKET_API_KEY", settings.get("api_key")) - if currency: - _add_override(overrides, "HELEKET_CURRENCY", currency) - _add_override(overrides, "HELEKET_SUPPORTED_CURRENCIES", currency) + if currency and currency != "RUB": + warnings.append( + f"Heleket source currency was {currency}; Minishop keeps payment currency " + "controlled by tariffs/default currency. Configure HELEKET_CURRENCY manually " + "if this instance needs a different default." + ) return _provider_mapping_result(gateway_type, ["heleket"], overrides, warnings) if gateway_type == "FREEKASSA": @@ -492,8 +498,12 @@ def remnashop_payment_gateway_overrides( _add_override(overrides, "PLATEGA_SECRET", settings.get("api_key")) _add_override(overrides, "PLATEGA_PAYMENT_METHOD", settings.get("payment_method")) _add_override(overrides, "PLATEGA_SBP_METHOD", settings.get("payment_method")) - if currency: - _add_override(overrides, "PLATEGA_SUPPORTED_CURRENCIES", currency) + if currency and currency != "RUB": + warnings.append( + f"Platega source currency was {currency}; Minishop keeps payment currency " + "controlled by tariffs/default currency. Configure PLATEGA_SUPPORTED_CURRENCIES " + "manually if this instance needs a different currency." + ) return _provider_mapping_result(gateway_type, ["platega_sbp"], overrides, warnings) return _provider_mapping_result(gateway_type, [], overrides, warnings) @@ -972,12 +982,15 @@ class RemnashopImporter: "REMNAWAVE_WEBHOOK_SECRET", "BOT_SUPPORT_USERNAME", "APP_DEFAULT_LOCALE", - "BOT_MINI_APP", ) if self.source_env.get(key) + and not _is_placeholder_setting_value(self.source_env.get(key)) ), "has_app_crypt_key": bool(self.source_env.get("APP_CRYPT_KEY")), "source_urls": remnashop_source_urls_from_env(self.source_env), + "ignored_keys_present": sorted( + key for key in ("BOT_MINI_APP",) if self.source_env.get(key) + ), }, ) return written @@ -1595,13 +1608,16 @@ class RemnashopImporter: "REMNAWAVE_WEBHOOK_SECRET", "BOT_SUPPORT_USERNAME", "APP_DEFAULT_LOCALE", - "BOT_MINI_APP", "APP_DOMAIN", "APP_CRYPT_KEY", ) if self.source_env.get(key) + and not _is_placeholder_setting_value(self.source_env.get(key)) ), "source_urls": remnashop_source_urls_from_env(self.source_env), + "ignored_keys_present": sorted( + key for key in ("BOT_MINI_APP",) if self.source_env.get(key) + ), }, } env_override_keys = await self.import_env_settings() diff --git a/docs/migrations/remnashop.md b/docs/migrations/remnashop.md index a719c4a..00dd2af 100644 --- a/docs/migrations/remnashop.md +++ b/docs/migrations/remnashop.md @@ -33,14 +33,30 @@ message logs как заметки, чтобы администратор мог - `REMNAWAVE_TOKEN` -> `PANEL_API_KEY`; - `REMNAWAVE_WEBHOOK_SECRET` -> `PANEL_WEBHOOK_SECRET`; - `BOT_SUPPORT_USERNAME` -> `SUPPORT_LINK`; -- `APP_DEFAULT_LOCALE` -> `DEFAULT_LANGUAGE`; -- `BOT_MINI_APP` -> `SUBSCRIPTION_MINI_APP_URL`, если там уже HTTPS URL. +- `APP_DEFAULT_LOCALE` -> `DEFAULT_LANGUAGE`. + +`BOT_MINI_APP` из Remnashop не переносится автоматически. В Remnashop эта +переменная управляет кнопкой подключения к subscription page или внешнему Mini +App, а не веб-кабинетом Remnashop. В Minishop `SUBSCRIPTION_MINI_APP_URL` +должен указывать на текущий frontend/Mini App этого стека; wizard настраивает +его из `WEBHOOK_HOST`/`MINIAPP_HOST` или `MINIAPP_PUBLIC_URL`. + +Значения-заглушки вроде `change_me` importer пропускает, чтобы случайно не +записать шаблонные секреты в рабочую конфигурацию. Платежные провайдеры берутся из таблицы Remnashop `payment_gateways`. Поддерживаются и автоматически маппятся: Telegram Stars, YooKassa, WATA, CryptoPay, Heleket, FreeKassa и Platega. Для них importer переносит флаги -включения, API-ключи/merchant IDs и доступные provider-specific параметры в -раздел настроек админки. +включения, API-ключи/merchant IDs и прямые технические параметры, без которых +провайдер не сможет работать: YooKassa receipt email/VAT, FreeKassa second +secret/payment method/server IP и Platega payment method. + +Provider currency и supported-currency ограничения не переносятся автоматически: +в Minishop валюта платежа управляется тарифами и `DEFAULT_CURRENCY_SYMBOL`. +Если старый gateway Remnashop был настроен на нестандартную валюту, importer +оставит предупреждение в JSON-сводке; проверьте `CRYPTOPAY_ASSET`, +`HELEKET_CURRENCY`, `HELEKET_SUPPORTED_CURRENCIES` или +`PLATEGA_SUPPORTED_CURRENCIES` вручную. Провайдеры YooMoney, Cryptomus, MulenPay, PayMaster, RoboKassa и UrlPay сейчас не имеют прямого аналога в Minishop. Если они были в Remnashop, importer @@ -72,7 +88,7 @@ Remnashop может хранить секреты в формате `enc_...`. `raw.githubusercontent.com`, без клонирования репозитория. 2. Вы указываете source PostgreSQL DSN Remnashop и schema, обычно `public`. 3. Опционально указываете путь к старому Remnashop `.env` для `APP_CRYPT_KEY`, - Remnawave API settings и переносимых payment/provider settings. + Remnawave API settings и переносимых settings. 4. Вы выбираете целевую БД: текущую compose-БД или ручной target DSN. 5. При необходимости указываете JSON map тарифов Remnashop в локальные `tariff_key`, например `{"basic": "standard_month"}`. diff --git a/tests/test_remnashop_import.py b/tests/test_remnashop_import.py index 0010ef1..4316f96 100644 --- a/tests/test_remnashop_import.py +++ b/tests/test_remnashop_import.py @@ -73,10 +73,24 @@ def test_remnashop_env_parser_and_overrides_map_safe_values(): "PANEL_WEBHOOK_SECRET": "panel secret", "SUPPORT_LINK": "https://t.me/support_bot", "DEFAULT_LANGUAGE": "en", - "SUBSCRIPTION_MINI_APP_URL": "https://app.example.com/", } +def test_remnashop_env_overrides_skip_placeholders_and_mini_app(): + overrides = remnashop_env_overrides( + { + "REMNAWAVE_HOST": "change_me", + "REMNAWAVE_TOKEN": "change_me", + "REMNAWAVE_WEBHOOK_SECRET": "change_me", + "BOT_SUPPORT_USERNAME": "change_me", + "APP_DEFAULT_LOCALE": "change_me", + "BOT_MINI_APP": "https://old-mini-app.example.com/", + } + ) + + assert overrides == {} + + def test_remnashop_yookassa_gateway_maps_to_current_provider_settings(): result = remnashop_payment_gateway_overrides( { @@ -138,6 +152,38 @@ def test_remnashop_free_kassa_and_platega_gateways_map_available_settings(): assert platega["provider_ids"] == ["platega_sbp"] assert platega["overrides"]["PLATEGA_SBP_ENABLED"] is True assert platega["overrides"]["PLATEGA_SBP_METHOD"] == 2 + assert "PLATEGA_SUPPORTED_CURRENCIES" not in platega["overrides"] + + +def test_remnashop_crypto_provider_currency_is_not_imported_as_setting_override(): + cryptopay = remnashop_payment_gateway_overrides( + { + "type": "CRYPTOPAY", + "currency": "USD", + "is_active": True, + "settings": {"api_key": "crypto-token"}, + } + ) + assert cryptopay["overrides"] == { + "CRYPTOPAY_ENABLED": True, + "CRYPTOPAY_TOKEN": "crypto-token", + } + assert any("source currency was USD" in warning for warning in cryptopay["warnings"]) + + heleket = remnashop_payment_gateway_overrides( + { + "type": "HELEKET", + "currency": "USD", + "is_active": True, + "settings": {"merchant_id": "merchant", "api_key": "secret"}, + } + ) + assert heleket["overrides"] == { + "HELEKET_ENABLED": True, + "HELEKET_MERCHANT_ID": "merchant", + "HELEKET_API_KEY": "secret", + } + assert "HELEKET_CURRENCY" not in heleket["overrides"] def test_remnashop_unsupported_gateway_is_reported_without_overrides(): From c4d2d0a2fe58e552c34f21f397fb54b421f11cf6 Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 15:26:37 +0300 Subject: [PATCH 09/11] fix: wrap long route contract line --- tests/test_webapp_route_contract.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_webapp_route_contract.py b/tests/test_webapp_route_contract.py index f1378ac..60f61d2 100644 --- a/tests/test_webapp_route_contract.py +++ b/tests/test_webapp_route_contract.py @@ -183,7 +183,10 @@ class WebAppRouteContractTests(unittest.TestCase): ("PATCH", "/api/admin/promos/{promo_id}"): "admin_promo_update_route", ("DELETE", "/api/admin/promos/{promo_id}"): "admin_promo_delete_route", ("GET", "/api/admin/logs"): "admin_logs_route", - ("GET", "/api/admin/broadcast/audience-counts"): "admin_broadcast_audience_counts_route", + ( + "GET", + "/api/admin/broadcast/audience-counts", + ): "admin_broadcast_audience_counts_route", ("POST", "/api/admin/broadcast"): "admin_broadcast_route", ("POST", "/api/admin/sync"): "admin_sync_route", ("GET", "/api/admin/ads"): "admin_ads_list_route", From 4d86015d25b0dda6446fe77124b3807e2f4fb3db Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 15:32:19 +0300 Subject: [PATCH 10/11] style: apply ruff formatting --- backend/bot/handlers/user/start.py | 6 ++---- backend/bot/services/promo_code_service.py | 2 +- backend/db/models.py | 4 +--- backend/scripts/__init__.py | 1 - backend/scripts/import_legacy.py | 3 +-- tests/test_install_script.py | 4 ++-- 6 files changed, 7 insertions(+), 13 deletions(-) diff --git a/backend/bot/handlers/user/start.py b/backend/bot/handlers/user/start.py index 8446546..5328a8c 100644 --- a/backend/bot/handlers/user/start.py +++ b/backend/bot/handlers/user/start.py @@ -473,12 +473,10 @@ async def ensure_required_channel_subscription( @router.message(CommandStart()) +@router.message(CommandStart(magic=F.args.regexp(r"^ref_([A-Za-z0-9_-]{1,64})$").as_("ref_match"))) @router.message( - CommandStart( - magic=F.args.regexp(r"^ref_([A-Za-z0-9_-]{1,64})$").as_("ref_match") - ) + CommandStart(magic=F.args.regexp(r"^promo_([A-Za-z0-9_-]{1,100})$").as_("promo_match")) ) -@router.message(CommandStart(magic=F.args.regexp(r"^promo_([A-Za-z0-9_-]{1,100})$").as_("promo_match"))) @router.message(CommandStart(magic=F.args.regexp(r"^admin_user_(\d+)$").as_("admin_user_match"))) @router.message(CommandStart(magic=F.args.regexp(r"^ticket_(\d+)$").as_("ticket_match"))) @router.message(CommandStart(magic=F.args.regexp(r"^notifications$").as_("notifications_match"))) diff --git a/backend/bot/services/promo_code_service.py b/backend/bot/services/promo_code_service.py index 6f6a63d..947550d 100644 --- a/backend/bot/services/promo_code_service.py +++ b/backend/bot/services/promo_code_service.py @@ -75,7 +75,7 @@ class PromoCodeService: "promo_code_too_many_attempts", seconds=throttle_result.retry_after or max(1, int(self.settings.BRUTE_FORCE_LOCK_SECONDS)), - ) + ) return False, _("promo_code_not_found", code=code_display) applied_code = str(promo_data.code or lookup_code) diff --git a/backend/db/models.py b/backend/db/models.py index e524d0d..585dcb4 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -409,9 +409,7 @@ class LegacyReferralCode(Base): user = relationship("User") - __table_args__ = ( - UniqueConstraint("source", "code", name="uq_legacy_referral_source_code"), - ) + __table_args__ = (UniqueConstraint("source", "code", name="uq_legacy_referral_source_code"),) class LegacyImportMapping(Base): diff --git a/backend/scripts/__init__.py b/backend/scripts/__init__.py index 7e7e6c0..51b3887 100644 --- a/backend/scripts/__init__.py +++ b/backend/scripts/__init__.py @@ -1,2 +1 @@ """Operational one-shot scripts shipped with the backend image.""" - diff --git a/backend/scripts/import_legacy.py b/backend/scripts/import_legacy.py index f3dbd0b..5936020 100644 --- a/backend/scripts/import_legacy.py +++ b/backend/scripts/import_legacy.py @@ -552,8 +552,7 @@ def remnashop_post_migration_actions( "new_url": _target_webhook_url(target_webhook_base_url, "/webhook/panel"), "where": "Remnawave Panel -> WEBHOOK_URL", "secret": ( - "Set the Remnawave webhook secret to the value stored in " - "PANEL_WEBHOOK_SECRET." + "Set the Remnawave webhook secret to the value stored in PANEL_WEBHOOK_SECRET." ), }, "payment_providers": payment_actions, diff --git a/tests/test_install_script.py b/tests/test_install_script.py index bc2477c..204db17 100644 --- a/tests/test_install_script.py +++ b/tests/test_install_script.py @@ -57,8 +57,8 @@ def test_shell_installer_only_prepares_data_mount_not_runtime_content(): script = INSTALL_SCRIPT.read_text(encoding="utf-8") assert 'data_dir="$TARGET_DIR/data"' in script - assert "mkdir -p \"$data_dir\"" in script - assert "chown \"$APP_UID:$APP_GID\" \"$data_dir\"" in script + assert 'mkdir -p "$data_dir"' in script + assert 'chown "$APP_UID:$APP_GID" "$data_dir"' in script assert "data_dir/themes" not in script assert "webapp-logo" not in script assert "webapp-emoji" not in script From 8e5124e4cfe19259cae3e800198538a06045d992 Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 15:57:04 +0300 Subject: [PATCH 11/11] style: apply prettier formatting to UserDetailModal --- frontend/src/admin/sections/UserDetailModal.svelte | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/src/admin/sections/UserDetailModal.svelte b/frontend/src/admin/sections/UserDetailModal.svelte index 35a36f2..c34339a 100644 --- a/frontend/src/admin/sections/UserDetailModal.svelte +++ b/frontend/src/admin/sections/UserDetailModal.svelte @@ -571,9 +571,7 @@ {#each openedUserDetail.subscriptions.slice(0, 8) as sub}
- {subscriptionDisplayLabel(sub)} + {subscriptionDisplayLabel(sub)} {at( "user_history_until",