Initial release: full Arma 3 mod management toolchain
Pipeline: parse HTML presets, compare modlists, download from Caddy file server, create junctions/symlinks to Arma 3 Server directory. Includes update/sync flows, missing-mod reporting, OS compat layer, shared config, dep checker, comprehensive test suite (71 tests). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
revernomad17
205
fetch_mods.py
Normal file
205
fetch_mods.py
Normal file
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
CLI entry point: download mods from the Caddy file server using comparison.json.
|
||||
|
||||
Folder layout produced:
|
||||
downloads/
|
||||
shared/ <- mods common to all presets
|
||||
@ace/
|
||||
@cba_a3/
|
||||
150th_MW_2026_v1.0/ <- preset-unique mods
|
||||
@rhsusaf/
|
||||
150th_WW2_2026_V1.0/
|
||||
@ifa3_aio/
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
from arma_modlist_tools.compat import fix_console_encoding
|
||||
from arma_modlist_tools.config import load_config
|
||||
from arma_modlist_tools.fetcher import (
|
||||
make_session,
|
||||
build_server_index,
|
||||
find_mod_folder,
|
||||
list_mod_files,
|
||||
download_file,
|
||||
)
|
||||
from arma_modlist_tools.reporter import build_missing_report, save_missing_report
|
||||
|
||||
fix_console_encoding()
|
||||
|
||||
|
||||
def _fmt_bytes(n: int) -> str:
|
||||
for unit in ("B", "KB", "MB", "GB"):
|
||||
if n < 1024:
|
||||
return f"{n:.1f} {unit}"
|
||||
n /= 1024
|
||||
return f"{n:.1f} TB"
|
||||
|
||||
|
||||
def _build_download_queue(comparison: dict) -> list[tuple[dict, str]]:
|
||||
"""Return flat list of (mod_entry, group_name) for every mod."""
|
||||
queue = []
|
||||
for mod in comparison["shared"]["mods"]:
|
||||
queue.append((mod, "shared"))
|
||||
for preset_name, data in comparison["unique"].items():
|
||||
for mod in data["mods"]:
|
||||
queue.append((mod, preset_name))
|
||||
return queue
|
||||
|
||||
|
||||
def main() -> None:
|
||||
cfg = load_config()
|
||||
|
||||
# ---- Load comparison ----
|
||||
if not cfg.comparison.exists():
|
||||
print(f"ERROR: {cfg.comparison} not found. Run compare_modlists.py first.")
|
||||
sys.exit(1)
|
||||
comparison = json.loads(cfg.comparison.read_text(encoding="utf-8"))
|
||||
queue = _build_download_queue(comparison)
|
||||
|
||||
print(f"Loaded comparison: {', '.join(comparison['compared_presets'])}")
|
||||
print(f"Total mods to consider: {len(queue)}\n")
|
||||
|
||||
# ---- Build server index ----
|
||||
print("Building server index (fetching meta.cpp for each server folder)...")
|
||||
index = build_server_index(cfg.server_url, cfg.server_auth)
|
||||
tqdm.write(f" Indexed {len(index['by_steam_id'])} mods by steam_id, "
|
||||
f"{len(index['by_name'])} by name\n")
|
||||
|
||||
# ---- Resolve mods against server ----
|
||||
print("Resolving mods against server index...")
|
||||
session = make_session(cfg.server_auth)
|
||||
resolved: list[tuple[dict, str, str]] = []
|
||||
not_found: list[dict] = []
|
||||
|
||||
for mod, group in queue:
|
||||
url = find_mod_folder(mod, index)
|
||||
if url:
|
||||
resolved.append((mod, group, url))
|
||||
else:
|
||||
not_found.append(mod)
|
||||
|
||||
tqdm.write(f" {len(resolved)} matched "
|
||||
f"{len(not_found)} not found on server\n")
|
||||
|
||||
# ---- Save missing report ----
|
||||
report = build_missing_report(comparison, index)
|
||||
save_missing_report(report, cfg.missing_report)
|
||||
if not_found:
|
||||
tqdm.write(f" Missing report saved: {cfg.missing_report} "
|
||||
f"({report['missing']} mods)\n")
|
||||
|
||||
if not resolved:
|
||||
print("Nothing to download.")
|
||||
sys.exit(0)
|
||||
|
||||
# ---- Pre-scan file lists ----
|
||||
print("Scanning server for file lists...")
|
||||
mod_file_lists: list[tuple[dict, str, str, list]] = []
|
||||
conflicts: list[str] = []
|
||||
|
||||
with tqdm(total=len(resolved), unit="mod", desc=" Scanning", leave=False) as bar:
|
||||
for mod, group, folder_url in resolved:
|
||||
folder_name = folder_url.rstrip("/").split("/")[-1]
|
||||
dest_path = cfg.downloads / group / folder_name
|
||||
files = list_mod_files(folder_url, session)
|
||||
mod_file_lists.append((mod, group, folder_url, files))
|
||||
if dest_path.exists():
|
||||
conflicts.append(str(dest_path))
|
||||
bar.update(1)
|
||||
|
||||
total_bytes = sum(size for _, _, _, files in mod_file_lists for _, _, size in files)
|
||||
tqdm.write(f" Total download size: {_fmt_bytes(total_bytes)}\n")
|
||||
|
||||
# ---- Conflict resolution (ask once) ----
|
||||
overwrite = False
|
||||
if conflicts:
|
||||
print(f"Conflict: {len(conflicts)} destination folder(s) already exist:")
|
||||
for c in conflicts[:5]:
|
||||
print(f" {c}")
|
||||
if len(conflicts) > 5:
|
||||
print(f" ... and {len(conflicts) - 5} more")
|
||||
while True:
|
||||
choice = input("\nOverwrite existing? [s]kip / [o]verwrite: ").strip().lower()
|
||||
if choice in ("s", "skip"):
|
||||
break
|
||||
if choice in ("o", "overwrite"):
|
||||
overwrite = True
|
||||
break
|
||||
print(" Please enter 's' or 'o'.")
|
||||
print()
|
||||
|
||||
# ---- Download ----
|
||||
print(f"Starting downloads: {len(mod_file_lists)} mods\n")
|
||||
|
||||
total_downloaded_bytes = total_downloaded_files = total_skipped_files = 0
|
||||
|
||||
with tqdm(
|
||||
total=len(mod_file_lists),
|
||||
unit="mod",
|
||||
desc="Overall",
|
||||
position=0,
|
||||
dynamic_ncols=True,
|
||||
) as mod_bar:
|
||||
for i, (mod, group, folder_url, files) in enumerate(mod_file_lists, 1):
|
||||
folder_name = folder_url.rstrip("/").split("/")[-1]
|
||||
dest_path = cfg.downloads / group / folder_name
|
||||
|
||||
tqdm.write(
|
||||
f"\n[{i}/{len(mod_file_lists)}] {folder_name}"
|
||||
f" -> {dest_path}/"
|
||||
f" (group: {group})"
|
||||
)
|
||||
|
||||
mod_bytes = mod_files_dl = mod_files_skip = 0
|
||||
|
||||
for rel, file_url, size in files:
|
||||
dest_file = dest_path / rel
|
||||
if dest_file.exists() and not overwrite:
|
||||
tqdm.write(f" SKIP {rel}")
|
||||
mod_files_skip += 1
|
||||
continue
|
||||
|
||||
with tqdm(
|
||||
total=size if size else None,
|
||||
unit="B", unit_scale=True, unit_divisor=1024,
|
||||
desc=f" {rel[-45:]:45s}",
|
||||
position=1, leave=False, dynamic_ncols=True,
|
||||
) as file_bar:
|
||||
n = download_file(
|
||||
file_url, dest_file, session,
|
||||
on_chunk=lambda b: file_bar.update(b),
|
||||
)
|
||||
|
||||
mod_bytes += n
|
||||
mod_files_dl += 1
|
||||
|
||||
total_downloaded_bytes += mod_bytes
|
||||
total_downloaded_files += mod_files_dl
|
||||
total_skipped_files += mod_files_skip
|
||||
|
||||
tqdm.write(
|
||||
f" Done {mod_files_dl} downloaded"
|
||||
+ (f" {mod_files_skip} skipped" if mod_files_skip else "")
|
||||
+ f" {_fmt_bytes(mod_bytes)}"
|
||||
)
|
||||
mod_bar.update(1)
|
||||
|
||||
# ---- Summary ----
|
||||
print(f"\n{'─' * 50}")
|
||||
print(f" Mods processed : {len(mod_file_lists)}")
|
||||
print(f" Files downloaded: {total_downloaded_files}")
|
||||
print(f" Files skipped : {total_skipped_files}")
|
||||
print(f" Mods not found : {len(not_found)}")
|
||||
print(f" Total downloaded: {_fmt_bytes(total_downloaded_bytes)}")
|
||||
print(f" Output root : {cfg.downloads.resolve()}")
|
||||
if not_found:
|
||||
print(f" Missing report : {cfg.missing_report}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user