48637ffe90
Before step_fetch, scan all downloads/ subdirs and move any mod that comparison.json now assigns to a different group. Matching uses steam_id (via meta.cpp publishedid) first, normalized name as fallback. Stale junctions in arma_dir are removed before the folder move so step_link can re-create them pointing to the new location. - New arma_modlist_tools/migrator.py: migrate_mod_groups() - run.py: step_migrate(), --skip-migrate flag, wired into dispatch loop - gui/app.py: step_migrate inserted as Step 3/5 between compare and fetch - gui/locales.py: add step3/4/5 names (en + vi), renumber old 3->4, 4->5 - test_suite.py: 7 new migrator tests (158 total, 0 failed)
256 lines
9.1 KiB
Python
256 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Orchestrator: run the full pipeline in sequence.
|
|
|
|
Steps:
|
|
1. parse — parse HTML presets -> modlist_json/
|
|
2. compare — compare presets -> comparison.json
|
|
3. fetch — download mods from server (also saves missing_report.json)
|
|
4. link — create junctions/symlinks to Arma 3 Server
|
|
|
|
Usage:
|
|
python run.py # full pipeline, all groups
|
|
python run.py --skip-fetch --skip-link # parse + compare only
|
|
python run.py --skip-parse --skip-compare --skip-fetch --group shared
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
|
|
from tqdm import tqdm
|
|
|
|
from arma_modlist_tools.compat import fix_console_encoding
|
|
from arma_modlist_tools.config import load_config
|
|
from arma_modlist_tools.parser import parse_modlist_dir
|
|
from arma_modlist_tools.compare import compare_presets
|
|
from arma_modlist_tools.fetcher import (
|
|
build_server_index, find_mod_folder,
|
|
list_mod_files, download_file, make_session,
|
|
)
|
|
from arma_modlist_tools.linker import link_group
|
|
from arma_modlist_tools.reporter import build_missing_report, save_missing_report
|
|
|
|
fix_console_encoding()
|
|
|
|
|
|
def _fmt_bytes(n: int) -> str:
|
|
for unit in ("B", "KB", "MB", "GB"):
|
|
if n < 1024:
|
|
return f"{n:.1f} {unit}"
|
|
n /= 1024
|
|
return f"{n:.1f} TB"
|
|
|
|
|
|
def _header(step: int, total: int, name: str) -> None:
|
|
print(f"\n{'='*56}")
|
|
print(f" Step {step}/{total}: {name}")
|
|
print(f"{'='*56}\n")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Step implementations
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def step_parse(cfg) -> None:
|
|
cfg.modlist_json.mkdir(exist_ok=True)
|
|
presets = parse_modlist_dir(cfg.modlist_html)
|
|
if not presets:
|
|
print(f" No .html files found in {cfg.modlist_html}/")
|
|
return
|
|
for preset in presets:
|
|
out = cfg.modlist_json / (preset["preset_name"] + ".json")
|
|
out.write_text(json.dumps(preset, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
print(f" {preset['source_file']} -> {out} ({preset['mod_count']} mods)")
|
|
|
|
|
|
def step_compare(cfg) -> None:
|
|
presets = parse_modlist_dir(cfg.modlist_html)
|
|
if len(presets) < 2:
|
|
print(" Need at least 2 preset files to compare.")
|
|
return
|
|
result = compare_presets(*presets)
|
|
cfg.modlist_json.mkdir(exist_ok=True)
|
|
cfg.comparison.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
print(f" Compared: {', '.join(result['compared_presets'])}")
|
|
print(f" Shared: {result['shared']['mod_count']} | ", end="")
|
|
print(" ".join(f"{k}: {v['mod_count']} unique" for k, v in result["unique"].items()))
|
|
print(f" -> {cfg.comparison}")
|
|
|
|
|
|
def step_migrate(cfg) -> None:
|
|
if not cfg.comparison.exists():
|
|
print(f" NOTE: {cfg.comparison} not found — skipping migration.")
|
|
return
|
|
if not cfg.downloads.is_dir():
|
|
print(f" NOTE: downloads dir missing ({cfg.downloads}) — nothing to migrate.")
|
|
return
|
|
from arma_modlist_tools.migrator import migrate_mod_groups
|
|
comparison = json.loads(cfg.comparison.read_text(encoding="utf-8"))
|
|
arma_dir = cfg.arma_dir if cfg.arma_dir.is_dir() else None
|
|
result = migrate_mod_groups(cfg.downloads, arma_dir, comparison)
|
|
if result["errors"]:
|
|
for name, err in result["errors"].items():
|
|
print(f" ERROR {name}: {err}")
|
|
|
|
|
|
def step_fetch(cfg) -> None:
|
|
if not cfg.comparison.exists():
|
|
print(f" ERROR: {cfg.comparison} not found. Run parse + compare first.")
|
|
return
|
|
|
|
comparison = json.loads(cfg.comparison.read_text(encoding="utf-8"))
|
|
queue: list[tuple[dict, str]] = []
|
|
for mod in comparison["shared"]["mods"]:
|
|
queue.append((mod, "shared"))
|
|
for preset_name, data in comparison["unique"].items():
|
|
for mod in data["mods"]:
|
|
queue.append((mod, preset_name))
|
|
|
|
def _index_progress(current: int, total: int, name: str) -> None:
|
|
if current == 1 or current % 25 == 0 or current == total:
|
|
print(f" Indexing {current}/{total}: {name}")
|
|
|
|
print(f" Building server index...")
|
|
index = build_server_index(cfg.server_url, cfg.server_auth, progress_fn=_index_progress)
|
|
print(f" Indexed {len(index['by_steam_id'])} mods by steam_id, "
|
|
f"{len(index['by_name'])} by name\n")
|
|
|
|
session = make_session(cfg.server_auth)
|
|
resolved = []
|
|
for mod, group in queue:
|
|
url = find_mod_folder(mod, index)
|
|
if url:
|
|
resolved.append((mod, group, url))
|
|
|
|
# Save missing report
|
|
report = build_missing_report(comparison, index)
|
|
save_missing_report(report, cfg.missing_report)
|
|
print(f" {len(resolved)} / {len(queue)} mods found on server")
|
|
if report["missing"]:
|
|
print(f" {report['missing']} missing — saved to {cfg.missing_report}")
|
|
|
|
if not resolved:
|
|
print(" Nothing to download.")
|
|
return
|
|
|
|
# Scan + download
|
|
print()
|
|
mod_file_lists = []
|
|
for mod, group, folder_url in resolved:
|
|
folder_name = folder_url.rstrip("/").split("/")[-1]
|
|
dest_path = cfg.downloads / group / folder_name
|
|
files = list_mod_files(folder_url, session)
|
|
mod_file_lists.append((mod, group, folder_url, dest_path, files))
|
|
|
|
total_bytes = 0
|
|
with tqdm(total=len(mod_file_lists), unit="mod", desc=" Fetching", position=0, dynamic_ncols=True) as mod_bar:
|
|
for mod, group, folder_url, dest_path, files in mod_file_lists:
|
|
folder_name = folder_url.rstrip("/").split("/")[-1]
|
|
mod_bytes = 0
|
|
for rel, file_url, size in files:
|
|
dest_file = dest_path / rel
|
|
if dest_file.exists():
|
|
continue
|
|
with tqdm(
|
|
total=size if size else None,
|
|
unit="B", unit_scale=True, unit_divisor=1024,
|
|
desc=f" {rel[-40:]:40s}",
|
|
position=1, leave=False, dynamic_ncols=True,
|
|
) as file_bar:
|
|
n = download_file(file_url, dest_file, session,
|
|
on_chunk=lambda b: file_bar.update(b))
|
|
mod_bytes += n
|
|
total_bytes += mod_bytes
|
|
mod_bar.update(1)
|
|
|
|
print(f"\n Done. {len(mod_file_lists)} mods {_fmt_bytes(total_bytes)}")
|
|
|
|
|
|
def step_link(cfg, groups: list[str]) -> None:
|
|
if not cfg.arma_dir.exists():
|
|
print(f" NOTE: Arma dir not found ({cfg.arma_dir}) — skipping link step.")
|
|
return
|
|
|
|
for group in groups:
|
|
group_dir = cfg.downloads / group
|
|
if not group_dir.is_dir():
|
|
print(f" SKIP {group} — folder not found")
|
|
continue
|
|
result = link_group(group_dir, cfg.arma_dir)
|
|
print(f" {group:<32} "
|
|
f"{result['linked']} linked, "
|
|
f"{result['already_linked']} already linked"
|
|
+ (f", {result['failed']} failed" if result["failed"] else ""))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Entry point
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def main() -> None:
|
|
cfg = load_config()
|
|
|
|
parser = argparse.ArgumentParser(description="Run the full mod management pipeline.")
|
|
parser.add_argument("--skip-parse", action="store_true")
|
|
parser.add_argument("--skip-compare", action="store_true")
|
|
parser.add_argument("--skip-migrate", action="store_true")
|
|
parser.add_argument("--skip-fetch", action="store_true")
|
|
parser.add_argument("--skip-link", action="store_true")
|
|
parser.add_argument("--group", "-g", metavar="GROUP",
|
|
help="Link only this group (default: all groups in downloads/)")
|
|
args = parser.parse_args()
|
|
|
|
steps = [
|
|
(not args.skip_parse, "Parse presets"),
|
|
(not args.skip_compare, "Compare presets"),
|
|
(not args.skip_migrate, "Migrate mods"),
|
|
(not args.skip_fetch, "Fetch mods"),
|
|
(not args.skip_link, "Link mods"),
|
|
]
|
|
active_count = sum(1 for run, _ in steps if run)
|
|
step_num = 0
|
|
|
|
if (args.skip_parse and args.skip_compare and args.skip_migrate
|
|
and args.skip_fetch and args.skip_link):
|
|
print("All steps skipped — nothing to do.")
|
|
sys.exit(0)
|
|
|
|
for (run, name), fn_args in zip(steps, [
|
|
(cfg,),
|
|
(cfg,),
|
|
(cfg,),
|
|
(cfg,),
|
|
None, # handled separately
|
|
]):
|
|
if not run:
|
|
continue
|
|
step_num += 1
|
|
_header(step_num, active_count, name)
|
|
|
|
if name == "Link mods":
|
|
if args.group:
|
|
groups = [args.group]
|
|
else:
|
|
groups = sorted(
|
|
p.name for p in cfg.downloads.iterdir()
|
|
if cfg.downloads.is_dir() and p.is_dir()
|
|
) if cfg.downloads.is_dir() else []
|
|
step_link(cfg, groups)
|
|
elif name == "Parse presets":
|
|
step_parse(cfg)
|
|
elif name == "Compare presets":
|
|
step_compare(cfg)
|
|
elif name == "Migrate mods":
|
|
step_migrate(cfg)
|
|
elif name == "Fetch mods":
|
|
step_fetch(cfg)
|
|
|
|
print(f"\n{'='*56}")
|
|
print(" Pipeline complete.")
|
|
print(f"{'='*56}\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|