commit 91a38b269bd98f8da3e176040254251ab27e17bf Author: revernomad17 Date: Tue Apr 7 16:04:36 2026 +0700 Initial release: full Arma 3 mod management toolchain Pipeline: parse HTML presets, compare modlists, download from Caddy file server, create junctions/symlinks to Arma 3 Server directory. Includes update/sync flows, missing-mod reporting, OS compat layer, shared config, dep checker, comprehensive test suite (71 tests). Co-Authored-By: Claude Sonnet 4.6 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d8edea4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# Credentials — never commit +config.json + +# Generated output +modlist_json/ +downloads/ + +# Python +__pycache__/ +*.py[cod] +*.pyo +*.pyd +.Python +*.egg-info/ +dist/ +build/ +.eggs/ + +# Virtual environments +venv/ +.venv/ +env/ + +# Editor / OS +.vscode/ +.idea/ +*.swp +*.swo +.DS_Store +Thumbs.db + +# Claude Code local settings +.claude/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..e0a6043 --- /dev/null +++ b/README.md @@ -0,0 +1,514 @@ +# arma-modlist-tools + +Python toolchain for managing Arma 3 mod presets: parse launcher exports, compare presets, download mods from a Caddy file server, and create junction/symlink links to an Arma 3 Server installation. + +--- + +## Table of Contents + +1. [Prerequisites](#prerequisites) +2. [Installation](#installation) +3. [Configuration](#configuration) +4. [Quick Start — Full Pipeline](#quick-start--full-pipeline) +5. [Individual Scripts](#individual-scripts) + - [check_deps.py](#check_depspy) + - [parse_modlist.py](#parse_modlistpy) + - [compare_modlists.py](#compare_modlistspy) + - [fetch_mods.py](#fetch_modspy) + - [link_mods.py](#link_modspy) + - [report_missing.py](#report_missingpy) + - [sync_missing.py](#sync_missingpy) + - [update_mods.py](#update_modspy) + - [run.py](#runpy) +6. [Folder Structure](#folder-structure) +7. [Moving to a New Device](#moving-to-a-new-device) +8. [Running Tests](#running-tests) + +--- + +## Prerequisites + +| Requirement | Version | Notes | +|-------------|---------|-------| +| Python | >= 3.11 | `python --version` | +| requests | any | `pip install requests` | +| tqdm | any | `pip install tqdm` | +| Windows or Linux | — | Windows uses junctions, Linux uses symlinks | + +Run the dep checker to confirm everything is ready: + +``` +python check_deps.py +``` + +--- + +## Installation + +```bash +# Clone the repo +git clone https://git.revoluxiant.io.vn/revernomad17/arma-modlist-tools.git +cd arma-modlist-tools + +# Install dependencies +pip install -r requirements.txt + +# Copy config template and fill in your credentials/paths +cp config.template.json config.json +``` + +Edit `config.json` — see [Configuration](#configuration) below. + +--- + +## Configuration + +All scripts read a single `config.json` in the project root. Copy the template and fill in your values: + +```json +{ + "server": { + "base_url": "https://your-caddy-server/arma3mods/", + "username": "your_username", + "password": "your_password" + }, + "paths": { + "arma_dir": "C:\\Path\\To\\Arma 3 Server", + "downloads": "downloads", + "modlist_html": "modlist_html", + "modlist_json": "modlist_json" + } +} +``` + +| Key | Description | +|-----|-------------| +| `server.base_url` | Root URL of your Caddy file server. Must have a trailing slash. | +| `server.username` | HTTP Basic Auth username. | +| `server.password` | HTTP Basic Auth password. | +| `paths.arma_dir` | Absolute path to the Arma 3 Server directory where links will be created. | +| `paths.downloads` | Where mod files are downloaded locally. Relative to project root. | +| `paths.modlist_html` | Folder containing exported Arma 3 Launcher preset `.html` files. | +| `paths.modlist_json` | Folder where parsed/compared JSON files are saved. | + +> **Note:** `config.json` is in `.gitignore` because it contains credentials. Never commit it. + +--- + +## Quick Start — Full Pipeline + +Place your Arma 3 Launcher preset exports (`.html`) into the `modlist_html/` folder, then run: + +```bash +python run.py +``` + +This runs all four steps in sequence: + +``` +Step 1/4: Parse presets — modlist_html/*.html -> modlist_json/*.json +Step 2/4: Compare presets — produces modlist_json/comparison.json +Step 3/4: Fetch mods — downloads from server -> downloads/ +Step 4/4: Link mods — creates junctions/symlinks in Arma 3 Server dir +``` + +### Skip flags + +```bash +python run.py --skip-fetch --skip-link # parse + compare only +python run.py --skip-parse --skip-compare --skip-fetch # link only +python run.py --skip-parse --skip-compare --skip-fetch --group shared # link one group only +``` + +| Flag | Skips | +|------|-------| +| `--skip-parse` | Step 1 (HTML parsing) | +| `--skip-compare` | Step 2 (preset comparison) | +| `--skip-fetch` | Step 3 (downloading) | +| `--skip-link` | Step 4 (linking) | +| `--group GROUP` | Link step: only link this one group (e.g. `shared`) | + +--- + +## Individual Scripts + +### check_deps.py + +Verify Python version and required packages before running anything else. + +```bash +python check_deps.py +``` + +``` + Python 3.11.9 OK + OS Windows + + requests 2.33.0 OK + tqdm 4.67.3 OK + + All checks passed. Ready to run. +``` + +--- + +### parse_modlist.py + +Parse all `.html` preset exports in `modlist_html/` and save each as JSON in `modlist_json/`. + +**How to get the HTML files:** In Arma 3 Launcher → Mods → Preset → right-click → Export to HTML. Place the exported files in `modlist_html/`. + +```bash +python parse_modlist.py +``` + +``` +150th_MW_2026_v1.0.html -> modlist_json/150th_MW_2026_v1.0.json (52 mods) +150th_WW2_2026_V1.0.html -> modlist_json/150th_WW2_2026_V1.0.json (48 mods) +``` + +Output JSON per preset: +```json +{ + "preset_name": "150th_MW_2026_v1.0", + "source_file": "150th_MW_2026_v1.0.html", + "mod_count": 52, + "mods": [ + { + "name": "CBA_A3", + "source": "steam", + "url": "https://steamcommunity.com/sharedfiles/filedetails/?id=450814997", + "steam_id": "450814997" + } + ] +} +``` + +--- + +### compare_modlists.py + +Compare all presets in `modlist_html/` and produce `modlist_json/comparison.json` with a shared/unique breakdown. + +```bash +python compare_modlists.py +``` + +``` +Compared: 150th_MW_2026_v1.0, 150th_WW2_2026_V1.0 + Shared mods : 28 + Unique to 150th_MW... : 24 + Unique to 150th_WW2...: 20 + -> modlist_json/comparison.json +``` + +Output `comparison.json` structure: +```json +{ + "compared_presets": ["Preset_A", "Preset_B"], + "shared": { + "mod_count": 28, + "mods": [...] + }, + "unique": { + "Preset_A": { "mod_count": 24, "mods": [...] }, + "Preset_B": { "mod_count": 20, "mods": [...] } + } +} +``` + +**Requires:** at least 2 `.html` files in `modlist_html/`. + +--- + +### fetch_mods.py + +Download all mods from the Caddy file server into `downloads/`, organized by group. + +```bash +python fetch_mods.py +``` + +``` +Loaded comparison: 150th_MW_2026_v1.0, 150th_WW2_2026_V1.0 +Total mods to consider: 72 + +Building server index... 87 mods indexed + +[1/70] @ace -> downloads/shared/@ace/ (group: shared) + addons/ace_common/... 5.2 MB [████████████] 100% + ... + Done 42 downloaded 8.2 MB + +Overwrite existing? [s]kip / [o]verwrite: s +``` + +- Mods shared across all presets go into `downloads/shared/` +- Preset-unique mods go into `downloads//` +- Saves `modlist_json/missing_report.json` listing any mods not found on server +- Prompts once if destination folders already exist + +**Requires:** `modlist_json/comparison.json` (run `compare_modlists.py` first). + +--- + +### link_mods.py + +Manage junction/symlink links between `downloads/` and the Arma 3 Server directory. + +#### Check status + +```bash +python link_mods.py status --group shared +``` + +``` + Group : shared + Path : downloads/shared + Arma : C:\...\Arma 3 Server + + Mod Status + ---------------------------------------------------------- + @ace [LINKED] + @cba_a3 [------] + + 1 / 2 linked +``` + +#### Create links + +```bash +python link_mods.py link --group shared +``` + +``` + Linking group: shared -> C:\...\Arma 3 Server + + [+] @ace linked + [=] @cba_a3 already linked +``` + +#### Remove links + +```bash +python link_mods.py unlink --group shared +``` + +Prompts for confirmation before removing links. Removing a link does **not** delete the mod files in `downloads/`. + +#### List available groups + +Omit `--group` to see all available groups: + +```bash +python link_mods.py status +``` + +> **Windows note:** Creates NTFS directory junctions (`mklink /J`). No administrator rights required. +> **Linux note:** Creates standard symlinks (`os.symlink`). + +--- + +### report_missing.py + +Check which mods from `comparison.json` are missing from the file server. Saves `modlist_json/missing_report.json`. + +```bash +python report_missing.py +``` + +``` +Checking server index... 87 mods indexed +Cross-referencing 72 required mods... + + Missing from server (2): + + steam_id Group Name + --------------- ----------------------------- ---------------------------------------- + 2648308937 150th_WW2_2026_V1.0 IFA3 AIO + 463939057 shared ACE3 + + 70 / 72 found on server + Report saved: modlist_json/missing_report.json +``` + +**Requires:** `modlist_json/comparison.json`. + +--- + +### sync_missing.py + +Re-check the server for mods that were previously missing and download any that have since been added. + +```bash +python sync_missing.py +``` + +``` +Loading missing report: 2 mods previously missing +Re-checking server index... 89 mods indexed + + Newly available: 1 mods + Still missing : 1 mods + + [+] @ace -> downloads/shared/ + Done 8.2 MB + + Missing report updated: 1 still missing + + Linking newly added mods... + shared 1 new linked, 27 already linked +``` + +**Flow:** +1. Loads `modlist_json/missing_report.json` +2. Re-checks server index for newly available mods +3. Downloads newly available mods to the correct group folder +4. Updates `missing_report.json` (removes mods now downloaded) +5. Runs linker for affected groups — existing links are safely skipped + +**Requires:** `modlist_json/missing_report.json` (run `report_missing.py` or `fetch_mods.py` first). + +--- + +### update_mods.py + +Re-download mod files that have changed on the server without changing the modlist structure (same mods, updated file contents/versions). + +Detection uses **file size comparison**: a file is considered stale if it is missing locally or its local size differs from the server-reported size. + +```bash +python update_mods.py # check all groups and mods +python update_mods.py --group shared # check one group only +python update_mods.py --mod @ace # check one specific mod +python update_mods.py --force # re-download all files regardless of size +python update_mods.py --force --group shared +``` + +``` +Building server index... 87 mods indexed + + Mode: size-check + Checking 28 mod folder(s)... + + [=] @cba_a3 shared 4 files up-to-date + [+] @ace shared 42 files 3 updated (8.2 MB) + [=] @rhsusaf 150th_MW 88 files up-to-date + + Total: 134 files checked, 3 updated, 8.2 MB downloaded +``` + +> **No re-linking needed.** Junctions/symlinks already point at the `downloads/` folders, so updated files are immediately visible to the Arma 3 Server. + +--- + +### run.py + +Orchestrator that chains all four pipeline steps. Described in [Quick Start](#quick-start--full-pipeline) above. + +--- + +## Folder Structure + +``` +arma-modlist-tools/ +| +|- arma_modlist_tools/ # Python package (library code) +| |- __init__.py # Public exports +| |- parser.py # HTML preset parser +| |- compare.py # Preset comparison +| |- fetcher.py # Caddy server downloader +| |- linker.py # Junction/symlink manager +| |- reporter.py # Missing-mod report builder +| |- config.py # config.json loader +| |- compat.py # OS detection + encoding fix +| +|- modlist_html/ # INPUT: put your .html preset exports here +| |- MyPreset_A.html +| |- MyPreset_B.html +| +|- modlist_json/ # OUTPUT: generated JSON files (gitignored) +| |- MyPreset_A.json +| |- MyPreset_B.json +| |- comparison.json +| |- missing_report.json +| +|- downloads/ # OUTPUT: downloaded mod files (gitignored) +| |- shared/ +| | |- @ace/ +| | |- @cba_a3/ +| |- MyPreset_A/ +| |- @rhsusaf/ +| +|- config.json # YOUR config (gitignored — contains credentials) +|- config.template.json # Template to copy from +|- requirements.txt +| +|- run.py # Orchestrator (parse + compare + fetch + link) +|- parse_modlist.py # Step 1 standalone +|- compare_modlists.py # Step 2 standalone +|- fetch_mods.py # Step 3 standalone +|- link_mods.py # Link management (status/link/unlink) +|- report_missing.py # Missing mod report +|- sync_missing.py # Sync newly available missing mods +|- update_mods.py # Re-download updated mod files +|- check_deps.py # Dependency checker +|- test_suite.py # Test suite +``` + +--- + +## Moving to a New Device + +1. **Clone the repo** on the new device: + ```bash + git clone https://git.revoluxiant.io.vn/revernomad17/arma-modlist-tools.git + cd arma-modlist-tools + ``` + +2. **Install dependencies:** + ```bash + pip install -r requirements.txt + ``` + +3. **Create your config:** + ```bash + cp config.template.json config.json + # Edit config.json with correct arma_dir and server credentials + ``` + +4. **Add your preset exports:** + Copy your `.html` files from the Arma 3 Launcher into `modlist_html/`. + +5. **Run the full pipeline:** + ```bash + python check_deps.py # verify everything is ready + python run.py # parse → compare → fetch → link + ``` + +> The `downloads/` folder can be large (several GB of mod files). On a second device you can either let `run.py` re-download everything, or copy the `downloads/` folder manually and run `python run.py --skip-fetch` to skip the download step and just create links. + +--- + +## Running Tests + +The test suite covers all modules with 71 tests. No network connection required. + +```bash +python test_suite.py +``` + +``` +------------------------------------------------------------ + compat 6 tests + config 5 tests + parser 9 tests + compare 8 tests + fetcher 19 tests (pure functions, no network) + reporter 8 tests + linker 12 tests (uses temp dirs) + __init__ 2 tests + integration 2 tests +------------------------------------------------------------ + Results: 71 passed, 0 failed, 0 skipped (71 total) +``` diff --git a/arma_modlist_tools/__init__.py b/arma_modlist_tools/__init__.py new file mode 100644 index 0000000..94177ec --- /dev/null +++ b/arma_modlist_tools/__init__.py @@ -0,0 +1,32 @@ +from .parser import parse_mod_entry, parse_modlist_html, parse_modlist_dir +from .compare import compare_presets +from .fetcher import ( + make_session, build_server_index, find_mod_folder, + list_mod_files, list_mod_updates, download_file, download_mod_folder, +) +from .linker import ( + get_mod_folders, get_link_status, create_junction, + remove_junction, link_group, unlink_group, +) +from .config import load_config, Config +from .compat import is_windows, is_linux, get_os_label, fix_console_encoding +from .reporter import build_missing_report, save_missing_report + +__all__ = [ + # parser + "parse_mod_entry", "parse_modlist_html", "parse_modlist_dir", + # compare + "compare_presets", + # fetcher + "make_session", "build_server_index", "find_mod_folder", + "list_mod_files", "list_mod_updates", "download_file", "download_mod_folder", + # linker + "get_mod_folders", "get_link_status", "create_junction", + "remove_junction", "link_group", "unlink_group", + # config + "load_config", "Config", + # compat + "is_windows", "is_linux", "get_os_label", "fix_console_encoding", + # reporter + "build_missing_report", "save_missing_report", +] diff --git a/arma_modlist_tools/compare.py b/arma_modlist_tools/compare.py new file mode 100644 index 0000000..5050f86 --- /dev/null +++ b/arma_modlist_tools/compare.py @@ -0,0 +1,79 @@ +""" +arma_modlist_tools.compare +~~~~~~~~~~~~~~~~~~~~~~~~~~ +Compare two or more Arma 3 mod presets (parsed by :mod:`arma_modlist_tools.parser`) +and produce a breakdown of shared and preset-unique mods. + +Typical usage:: + + from arma_modlist_tools.parser import parse_modlist_dir + from arma_modlist_tools.compare import compare_presets + + presets = parse_modlist_dir("modlist_html") + result = compare_presets(*presets) +""" + +from __future__ import annotations + + +def _mod_key(mod: dict) -> str: + """Return the identity key for a mod. + + Uses ``steam_id`` when available (canonical Workshop identifier), + falls back to ``name`` for local mods that have no Workshop ID. + """ + return mod["steam_id"] or mod["name"] + + +def compare_presets(*presets: dict) -> dict: + """ + Compare two or more preset dicts and return a comparison dict. + + :param presets: Two or more preset dicts as returned by + :func:`~arma_modlist_tools.parser.parse_modlist_html`. + :returns: Dict with keys: + + - ``compared_presets`` — list of preset names that were compared + - ``shared`` — mods present in **every** preset + - ``mod_count`` — number of shared mods + - ``mods`` — list of mod entry dicts + - ``unique`` — per-preset mods not present in any other preset + - keyed by ``preset_name`` + - each value has ``mod_count`` and ``mods`` + + :raises ValueError: If fewer than two presets are provided. + """ + if len(presets) < 2: + raise ValueError("compare_presets requires at least two presets") + + # Build per-preset {identity_key -> mod_entry} mappings + preset_maps: list[dict[str, dict]] = [ + {_mod_key(mod): mod for mod in preset["mods"]} + for preset in presets + ] + + # Shared keys = intersection across ALL presets + shared_keys: set[str] = set(preset_maps[0].keys()) + for pm in preset_maps[1:]: + shared_keys &= pm.keys() + + # Shared mods: take entries from the first preset (identical across all) + shared_mods = [preset_maps[0][k] for k in preset_maps[0] if k in shared_keys] + + # Unique mods per preset: entries whose key is not in the shared set + unique: dict[str, dict] = {} + for preset, pm in zip(presets, preset_maps): + unique_mods = [mod for k, mod in pm.items() if k not in shared_keys] + unique[preset["preset_name"]] = { + "mod_count": len(unique_mods), + "mods": unique_mods, + } + + return { + "compared_presets": [p["preset_name"] for p in presets], + "shared": { + "mod_count": len(shared_mods), + "mods": shared_mods, + }, + "unique": unique, + } diff --git a/arma_modlist_tools/compat.py b/arma_modlist_tools/compat.py new file mode 100644 index 0000000..5aadb93 --- /dev/null +++ b/arma_modlist_tools/compat.py @@ -0,0 +1,108 @@ +""" +arma_modlist_tools.compat +~~~~~~~~~~~~~~~~~~~~~~~~~ +OS detection and cross-platform utilities shared by all CLI scripts. + +Supported platforms: + - Windows / Windows Server (sys.platform == "win32") + - Ubuntu / Ubuntu Server (sys.platform == "linux") + +Typical usage:: + + from arma_modlist_tools.compat import is_windows, get_os_label, fix_console_encoding + + fix_console_encoding() # call once at script start on Windows + print(get_os_label()) # "Windows Server", "Ubuntu", etc. +""" + +from __future__ import annotations + +import io +import platform +import sys + + +# --------------------------------------------------------------------------- +# Platform detection +# --------------------------------------------------------------------------- + +def is_windows() -> bool: + """Return ``True`` on Windows and Windows Server.""" + return sys.platform == "win32" + + +def is_linux() -> bool: + """Return ``True`` on Linux (Ubuntu, Ubuntu Server, and other distros).""" + return sys.platform == "linux" + + +def get_os_label() -> str: + """ + Return a human-readable OS label. + + Possible values: ``"Windows"``, ``"Windows Server"``, ``"Ubuntu"``, + ``"Ubuntu Server"``, ``"Linux"``, ``"Unknown"``. + """ + if is_windows(): + ver = platform.version() + # Windows Server versions contain "Server" in the version string + # e.g. "10.0.17763 ... Windows Server 2019 ..." + if "Server" in platform.version() or "Server" in platform.uname().version: + return "Windows Server" + return "Windows" + + if is_linux(): + # Read /etc/os-release for distro name + os_release = _read_os_release() + name = os_release.get("NAME", "").lower() + + if "ubuntu" in name: + # Distinguish desktop vs server: server images have no display server + if _is_headless(): + return "Ubuntu Server" + return "Ubuntu" + + return "Linux" + + return "Unknown" + + +def _read_os_release() -> dict[str, str]: + """Parse /etc/os-release into a dict (Linux only).""" + result: dict[str, str] = {} + try: + with open("/etc/os-release", encoding="utf-8") as f: + for line in f: + line = line.strip() + if "=" in line and not line.startswith("#"): + k, _, v = line.partition("=") + result[k] = v.strip('"') + except OSError: + pass + return result + + +def _is_headless() -> bool: + """Return True if no graphical display server is detected (headless/server).""" + import os + # Check for common display environment variables + return not (os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")) + + +# --------------------------------------------------------------------------- +# Console encoding +# --------------------------------------------------------------------------- + +def fix_console_encoding() -> None: + """ + Force UTF-8 output on Windows terminals that default to cp1252. + + Call once at the top of any CLI script that uses Unicode characters + (checkmarks, arrows, etc.). No-op on Linux. + """ + if not is_windows(): + return + if sys.stdout.encoding and sys.stdout.encoding.lower() == "utf-8": + return + sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace") + sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace") diff --git a/arma_modlist_tools/config.py b/arma_modlist_tools/config.py new file mode 100644 index 0000000..90815b7 --- /dev/null +++ b/arma_modlist_tools/config.py @@ -0,0 +1,109 @@ +""" +arma_modlist_tools.config +~~~~~~~~~~~~~~~~~~~~~~~~~ +Load and expose project configuration from ``config.json``. + +Search order for the config file: + 1. Explicit path passed to :func:`load_config` + 2. ``config.json`` in the current working directory + 3. ``config.json`` two levels above this module (project root) + +Typical usage:: + + from arma_modlist_tools.config import load_config + + cfg = load_config() + print(cfg.server_url) + print(cfg.arma_dir) +""" + +from __future__ import annotations + +import json +from pathlib import Path + + +class Config: + """Typed wrapper around the parsed ``config.json`` dict.""" + + def __init__(self, data: dict) -> None: + # Validate required keys immediately so callers get a clear error at + # load time rather than a confusing AttributeError deep in the pipeline. + _ = data["server"]["base_url"] + _ = data["server"]["username"] + _ = data["server"]["password"] + _ = data["paths"]["arma_dir"] + _ = data["paths"]["downloads"] + _ = data["paths"]["modlist_html"] + _ = data["paths"]["modlist_json"] + self._data = data + + # ---- server ---- + + @property + def server_url(self) -> str: + return self._data["server"]["base_url"] + + @property + def server_auth(self) -> tuple[str, str]: + return (self._data["server"]["username"], self._data["server"]["password"]) + + # ---- paths ---- + + @property + def arma_dir(self) -> Path: + return Path(self._data["paths"]["arma_dir"]) + + @property + def downloads(self) -> Path: + return Path(self._data["paths"]["downloads"]) + + @property + def modlist_html(self) -> Path: + return Path(self._data["paths"]["modlist_html"]) + + @property + def modlist_json(self) -> Path: + return Path(self._data["paths"]["modlist_json"]) + + # ---- derived paths ---- + + @property + def comparison(self) -> Path: + return self.modlist_json / "comparison.json" + + @property + def missing_report(self) -> Path: + return self.modlist_json / "missing_report.json" + + +def load_config(path: Path | str | None = None) -> Config: + """ + Load ``config.json`` and return a :class:`Config` instance. + + :param path: Explicit path to the config file. If ``None``, the function + searches the current working directory then the project root. + :raises FileNotFoundError: If no config file can be located. + :raises KeyError: If required keys are absent from the config file. + """ + if path is not None: + config_path = Path(path) + else: + # Try CWD first, then project root (two levels above this file) + cwd_path = Path.cwd() / "config.json" + root_path = Path(__file__).parent.parent / "config.json" + if cwd_path.exists(): + config_path = cwd_path + elif root_path.exists(): + config_path = root_path + else: + raise FileNotFoundError( + "config.json not found. " + f"Looked in:\n {cwd_path}\n {root_path}\n" + "Create config.json in the project root (copy from the template)." + ) + + with open(config_path, encoding="utf-8") as f: + data = json.load(f) + + return Config(data) diff --git a/arma_modlist_tools/fetcher.py b/arma_modlist_tools/fetcher.py new file mode 100644 index 0000000..5f21092 --- /dev/null +++ b/arma_modlist_tools/fetcher.py @@ -0,0 +1,268 @@ +""" +arma_modlist_tools.fetcher +~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download Arma 3 mods from a Caddy file server using a comparison dict as input. + +The server is expected to host mods as ``@ModName/`` folders under a base URL, +with a ``meta.cpp`` file inside each folder containing the Steam Workshop ID:: + + publishedid = 463939057; + +Typical usage:: + + from arma_modlist_tools.fetcher import ( + make_session, build_server_index, find_mod_folder, + list_mod_files, download_file, download_mod_folder, + ) + + session = make_session(("user", "password")) + index = build_server_index("https://example.com/arma3mods/", ("user", "pass")) + url = find_mod_folder({"steam_id": "463939057", "name": "ace"}, index) + files = list_mod_files(url, session) + download_mod_folder(url, Path("downloads/shared/@ace"), session) +""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from pathlib import Path + +import requests + +_CHUNK_SIZE = 64 * 1024 # 64 KB per read +_META_CPP_RE = re.compile(r"publishedid\s*=\s*(\d+)", re.IGNORECASE) +_NON_ALNUM_RE = re.compile(r"[^a-z0-9]") + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +def _list_dir(url: str, session: requests.Session) -> list[dict]: + """ + Fetch a Caddy browse directory listing as JSON. + Caddy returns a list of ``{name, size, url, is_dir, ...}`` dicts when the + ``Accept: application/json`` header is sent. + """ + resp = session.get(url, headers={"Accept": "application/json"}, timeout=30) + resp.raise_for_status() + data = resp.json() + # Caddy v2 returns a plain list; guard against wrapped responses + if isinstance(data, list): + return data + return data.get("items", []) + + +def _parse_meta_cpp(text: str) -> str | None: + """Extract ``publishedid`` from a ``meta.cpp`` file, or return ``None``.""" + m = _META_CPP_RE.search(text) + return m.group(1) if m else None + + +def _normalize_name(name: str) -> str: + """Strip leading ``@``, lowercase, remove all non-alphanumeric characters.""" + return _NON_ALNUM_RE.sub("", name.lower().lstrip("@")) + + +def _folder_url(base: str, name: str) -> str: + """Build a canonical trailing-slash folder URL.""" + return base.rstrip("/") + "/" + name.strip("/") + "/" + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def make_session(auth: tuple[str, str]) -> requests.Session: + """Return a ``requests.Session`` pre-configured with basic auth credentials.""" + s = requests.Session() + s.auth = auth + return s + + +def build_server_index(base_url: str, auth: tuple[str, str]) -> dict: + """ + Scan the root of the file server and build mod lookup maps. + + For every ``@...`` folder found at *base_url*, the function attempts to + fetch ``meta.cpp`` to extract the Steam Workshop ID. + + :param base_url: Root URL of the Caddy file server (trailing slash optional). + :param auth: ``(username, password)`` tuple for HTTP Basic Auth. + :returns: Dict with keys: + + - ``by_steam_id`` — ``{steam_id: folder_url}`` + - ``by_name`` — ``{normalized_name: folder_url}`` + - ``folders`` — raw list of item dicts from the root listing + """ + session = make_session(auth) + root = base_url.rstrip("/") + "/" + items = _list_dir(root, session) + folders = [it for it in items if it.get("is_dir")] + + by_steam_id: dict[str, str] = {} + by_name: dict[str, str] = {} + + for folder in folders: + name = folder["name"].strip("/") + url = _folder_url(root, name) + by_name[_normalize_name(name)] = url + + try: + resp = session.get(url + "meta.cpp", timeout=10) + if resp.ok: + sid = _parse_meta_cpp(resp.text) + if sid: + by_steam_id[sid] = url + except requests.RequestException: + pass # meta.cpp missing or unreachable — name-based fallback still works + + return {"by_steam_id": by_steam_id, "by_name": by_name, "folders": folders} + + +def find_mod_folder(mod: dict, index: dict) -> str | None: + """ + Return the server folder URL for a mod entry, or ``None`` if not found. + + Lookup order: + + 1. ``steam_id`` → ``index["by_steam_id"]`` (exact, reliable) + 2. Normalized ``name`` → ``index["by_name"]`` (fuzzy fallback for local mods) + + :param mod: Mod entry dict with at least ``"steam_id"`` and ``"name"`` keys. + :param index: Index dict returned by :func:`build_server_index`. + """ + if mod.get("steam_id"): + url = index["by_steam_id"].get(mod["steam_id"]) + if url: + return url + return index["by_name"].get(_normalize_name(mod.get("name", ""))) + + +def list_mod_files( + folder_url: str, + session: requests.Session, +) -> list[tuple[str, str, int]]: + """ + Recursively list all files under a mod folder on the server. + + :returns: List of ``(relative_path, absolute_url, size_bytes)`` tuples, + where *relative_path* is relative to *folder_url*. + """ + return _walk(folder_url.rstrip("/") + "/", session, "") + + +def list_mod_updates( + folder_url: str, + dest_path: Path, + session: requests.Session, +) -> list[tuple[str, str, int]]: + """ + Return only the files that are missing locally or whose local size differs + from the server size. Files that exist and match the server size are + considered up-to-date and omitted. + + Use this to detect which files need to be re-downloaded after the server + has been updated without changing the modlist structure. + + :param folder_url: Server folder URL for the mod (e.g. ``https://…/@ace/``). + :param dest_path: Local destination directory for this mod. + :param session: Authenticated ``requests.Session``. + :returns: Subset of :func:`list_mod_files` results — ``(rel_path, url, size)``. + """ + stale = [] + for rel, url, server_size in list_mod_files(folder_url, session): + local = dest_path / rel + if not local.exists(): + stale.append((rel, url, server_size)) + elif server_size and local.stat().st_size != server_size: + stale.append((rel, url, server_size)) + return stale + + +def _walk(url: str, session: requests.Session, prefix: str) -> list[tuple[str, str, int]]: + items = _list_dir(url, session) + result = [] + for item in items: + name = item["name"].strip("/") + rel = (prefix + "/" + name).lstrip("/") + item_url = url.rstrip("/") + "/" + name + if item.get("is_dir"): + result.extend(_walk(item_url + "/", session, rel)) + else: + result.append((rel, item_url, item.get("size", 0))) + return result + + +def download_file( + url: str, + dest: Path, + session: requests.Session, + on_chunk: Callable[[int], None] | None = None, +) -> int: + """ + Stream-download a single file to *dest*. + + :param on_chunk: Optional callback ``(bytes_written)`` called after each + chunk is flushed to disk. + :returns: Total bytes written. + """ + dest.parent.mkdir(parents=True, exist_ok=True) + resp = session.get(url, stream=True, timeout=120) + resp.raise_for_status() + written = 0 + with open(dest, "wb") as fh: + for chunk in resp.iter_content(chunk_size=_CHUNK_SIZE): + if chunk: + fh.write(chunk) + written += len(chunk) + if on_chunk: + on_chunk(len(chunk)) + return written + + +def download_mod_folder( + folder_url: str, + dest_path: Path, + session: requests.Session, + overwrite: bool = False, + on_file: Callable[[str, int, bool], None] | None = None, + on_chunk: Callable[[int], None] | None = None, +) -> dict: + """ + Recursively download all files in a mod folder. + + :param folder_url: Server folder URL (must be browsable by Caddy). + :param dest_path: Local destination directory (created if necessary). + :param session: Authenticated ``requests.Session``. + :param overwrite: If ``False``, existing files are skipped. + :param on_file: ``(rel_path, size_bytes, is_skipped)`` — called before + each file, whether it will be downloaded or skipped. + :param on_chunk: ``(bytes)`` — called per chunk **only** for files that + are actually downloaded (not skipped). + :returns: ``{"files_downloaded": n, "files_skipped": n, "bytes_downloaded": n}`` + """ + files = list_mod_files(folder_url, session) + downloaded = skipped = total_bytes = 0 + + for rel, url, size in files: + dest_file = dest_path / rel + is_skipped = dest_file.exists() and not overwrite + + if on_file: + on_file(rel, size, is_skipped) + + if is_skipped: + skipped += 1 + continue + + n = download_file(url, dest_file, session, on_chunk=on_chunk) + total_bytes += n + downloaded += 1 + + return { + "files_downloaded": downloaded, + "files_skipped": skipped, + "bytes_downloaded": total_bytes, + } diff --git a/arma_modlist_tools/linker.py b/arma_modlist_tools/linker.py new file mode 100644 index 0000000..b5ef454 --- /dev/null +++ b/arma_modlist_tools/linker.py @@ -0,0 +1,188 @@ +""" +arma_modlist_tools.linker +~~~~~~~~~~~~~~~~~~~~~~~~~ +Manage directory links between downloaded mod folders and the Arma 3 Server +directory. Works on both Windows (junction links) and Linux (symlinks). + +Platform behaviour: + - **Windows**: junctions via ``cmd /c mklink /J`` — no admin rights required. + - **Linux**: symlinks via ``os.symlink()`` — standard directory symlinks. + +Typical usage:: + + from arma_modlist_tools.linker import get_link_status, link_group, unlink_group + from pathlib import Path + + arma = Path("/opt/arma3server") + group = Path("downloads/shared") + + status = get_link_status(group, arma) + result = link_group(group, arma) +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +from .compat import is_windows + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +def _is_junction(path: Path) -> bool: + """ + Return ``True`` if *path* is an active directory junction / symlink. + + - **Windows**: checks ``FILE_ATTRIBUTE_REPARSE_POINT`` (0x400) in + ``os.lstat().st_file_attributes``. ``os.path.islink()`` is unreliable + for junctions on Windows. + - **Linux**: ``os.path.islink()`` correctly identifies symlinks. + """ + try: + if is_windows(): + s = os.lstat(str(path)) + attrs = getattr(s, "st_file_attributes", 0) + return bool(attrs & 0x400) # FILE_ATTRIBUTE_REPARSE_POINT + else: + return os.path.islink(str(path)) + except OSError: + return False + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def get_mod_folders(group_dir: Path) -> list[Path]: + """ + Return a sorted list of ``@*`` subdirectories inside *group_dir*. + + :param group_dir: The mod category folder (e.g. ``downloads/shared``). + """ + if not group_dir.is_dir(): + return [] + return sorted( + p for p in group_dir.iterdir() + if p.is_dir() and p.name.startswith("@") + ) + + +def get_link_status(group_dir: Path, arma_dir: Path) -> list[dict]: + """ + Return the link status for every ``@Mod`` folder in *group_dir*. + + :returns: List of dicts with keys: + + - ``name`` — folder name (e.g. ``@ace``) + - ``source_path`` — absolute path of the mod folder + - ``link_path`` — where the link would/does live in *arma_dir* + - ``is_linked`` — ``True`` if a junction/symlink currently exists + """ + result = [] + for mod in get_mod_folders(group_dir): + link_path = arma_dir / mod.name + result.append({ + "name": mod.name, + "source_path": mod.resolve(), + "link_path": link_path, + "is_linked": _is_junction(link_path), + }) + return result + + +def create_junction(link_path: Path, target: Path) -> bool: + """ + Create a directory junction (Windows) or symlink (Linux) at *link_path* + pointing to *target*. + + :returns: ``True`` on success, ``False`` on failure. + """ + if is_windows(): + proc = subprocess.run( + ["cmd", "/c", "mklink", "/J", str(link_path), str(target)], + capture_output=True, + text=True, + ) + return proc.returncode == 0 + else: + try: + os.symlink(str(target), str(link_path)) + return True + except OSError: + return False + + +def remove_junction(link_path: Path) -> tuple[bool, str]: + """ + Remove the junction / symlink at *link_path*. + + - **Windows**: ``os.rmdir()`` removes the junction pointer without touching + the target directory's contents. + - **Linux**: ``os.unlink()`` removes the symlink without touching the target. + + :returns: ``(True, "")`` on success, ``(False, error_message)`` on failure. + """ + try: + if is_windows(): + os.rmdir(str(link_path)) + else: + os.unlink(str(link_path)) + return True, "" + except OSError as exc: + return False, str(exc) + + +def link_group(group_dir: Path, arma_dir: Path) -> dict: + """ + Create links for every unlinked ``@Mod`` in *group_dir*. + + :returns: ``{"linked": n, "already_linked": n, "failed": n, "errors": {name: msg}}`` + """ + status = get_link_status(group_dir, arma_dir) + linked = already_linked = failed = 0 + errors: dict[str, str] = {} + + for s in status: + if s["is_linked"]: + already_linked += 1 + continue + if s["link_path"].exists(): + failed += 1 + errors[s["name"]] = "path exists but is not a junction/symlink" + continue + ok = create_junction(s["link_path"], s["source_path"]) + if ok: + linked += 1 + else: + failed += 1 + errors[s["name"]] = "link creation failed" + + return {"linked": linked, "already_linked": already_linked, "failed": failed, "errors": errors} + + +def unlink_group(group_dir: Path, arma_dir: Path) -> dict: + """ + Remove links for every linked ``@Mod`` in *group_dir*. + + :returns: ``{"unlinked": n, "not_linked": n, "failed": n, "errors": {name: msg}}`` + """ + status = get_link_status(group_dir, arma_dir) + unlinked = not_linked = failed = 0 + errors: dict[str, str] = {} + + for s in status: + if not s["is_linked"]: + not_linked += 1 + continue + ok, err = remove_junction(s["link_path"]) + if ok: + unlinked += 1 + else: + failed += 1 + errors[s["name"]] = err + + return {"unlinked": unlinked, "not_linked": not_linked, "failed": failed, "errors": errors} diff --git a/arma_modlist_tools/parser.py b/arma_modlist_tools/parser.py new file mode 100644 index 0000000..7e18617 --- /dev/null +++ b/arma_modlist_tools/parser.py @@ -0,0 +1,146 @@ +""" +arma_modlist_tools.parser +~~~~~~~~~~~~~~~~~~~~~~~~~ +Parse Arma 3 Launcher mod preset HTML files (.html exported from the launcher) +into plain Python dicts / lists suitable for JSON serialisation. + +Typical usage:: + + from arma_modlist_tools.parser import parse_modlist_html, parse_modlist_dir + + # single file + preset = parse_modlist_html("modlist_html/my_preset.html") + + # whole folder + presets = parse_modlist_dir("modlist_html") +""" + +import re +import xml.etree.ElementTree as ET +from pathlib import Path + + +# --------------------------------------------------------------------------- +# Public types (plain dicts — keep it dependency-free) +# --------------------------------------------------------------------------- + +# ModEntry: +# name : str display name from the launcher +# source : "steam" | "local" | "unknown" +# url : str | None full workshop / local path URL +# steam_id : str | None numeric workshop item ID extracted from the URL + +# Preset: +# preset_name : str stem of the source filename +# source_file : str basename of the source filename +# mod_count : int +# mods : list[ModEntry] + + +# --------------------------------------------------------------------------- +# Low-level helpers +# --------------------------------------------------------------------------- + +_STEAM_ID_RE = re.compile(r"[?&]id=(\d+)") + + +def _extract_steam_id(url: str) -> str | None: + """Return the numeric workshop item ID from a Steam URL, or None.""" + m = _STEAM_ID_RE.search(url) + return m.group(1) if m else None + + +def _source_from_class(css_class: str) -> str: + """Map a span CSS class to a source label.""" + if "from-steam" in css_class: + return "steam" + if "from-local" in css_class: + return "local" + return "unknown" + + +# --------------------------------------------------------------------------- +# Core parsing +# --------------------------------------------------------------------------- + +def parse_mod_entry(tr_element: ET.Element) -> dict | None: + """ + Parse a single ```` element into a mod dict. + + Returns ``None`` if the element does not contain a display name (i.e. it + is not a valid mod row). + """ + name: str | None = None + source: str = "unknown" + url: str | None = None + steam_id: str | None = None + + for td in tr_element: + dtype = td.get("data-type") + + if dtype == "DisplayName": + name = (td.text or "").strip() + continue + + for span in td.iter("span"): + css = span.get("class", "") + if "from-" in css: + source = _source_from_class(css) + + for a in td.iter("a"): + if a.get("data-type") == "Link": + href = (a.get("href") or "").strip() + if href: + url = href + steam_id = _extract_steam_id(href) + + if name is None: + return None + + return {"name": name, "source": source, "url": url, "steam_id": steam_id} + + +def parse_modlist_html(filepath: str | Path) -> dict: + """ + Parse an Arma 3 Launcher preset HTML file and return a preset dict. + + :param filepath: Path to the ``.html`` preset file. + :returns: Dict with keys ``preset_name``, ``source_file``, ``mod_count``, + and ``mods`` (list of mod entry dicts). + :raises FileNotFoundError: If *filepath* does not exist. + :raises ET.ParseError: If the file is not valid XML/HTML. + """ + path = Path(filepath) + tree = ET.parse(path) + root = tree.getroot() + + mods = [] + for tr in root.iter("tr"): + if tr.get("data-type") != "ModContainer": + continue + entry = parse_mod_entry(tr) + if entry is not None: + mods.append(entry) + + return { + "preset_name": path.stem, + "source_file": path.name, + "mod_count": len(mods), + "mods": mods, + } + + +def parse_modlist_dir(directory: str | Path) -> list[dict]: + """ + Parse all ``.html`` preset files in *directory* and return a list of + preset dicts (one per file, sorted by filename). + + :param directory: Folder containing ``.html`` preset files. + :returns: List of preset dicts as returned by :func:`parse_modlist_html`. + :raises NotADirectoryError: If *directory* does not exist or is not a dir. + """ + d = Path(directory) + if not d.is_dir(): + raise NotADirectoryError(f"Not a directory: {d}") + + return [parse_modlist_html(f) for f in sorted(d.glob("*.html"))] diff --git a/arma_modlist_tools/reporter.py b/arma_modlist_tools/reporter.py new file mode 100644 index 0000000..e513b40 --- /dev/null +++ b/arma_modlist_tools/reporter.py @@ -0,0 +1,95 @@ +""" +arma_modlist_tools.reporter +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Build and persist a report of mods that are required by the modlists but +absent from the file server. + +The report includes a ``group`` field per missing mod so downstream tools +(``sync_missing.py``) know exactly where to place it when it becomes +available on the server, without needing to re-read ``comparison.json``. + +Typical usage:: + + from arma_modlist_tools.reporter import build_missing_report, save_missing_report + + report = build_missing_report(comparison, server_index) + save_missing_report(report, cfg.missing_report) +""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path + + +def build_missing_report(comparison: dict, server_index: dict) -> dict: + """ + Cross-reference every mod in *comparison* against *server_index* and + return a report of mods that are not on the server. + + :param comparison: Dict as returned by :func:`~arma_modlist_tools.compare.compare_presets`. + :param server_index: Dict as returned by :func:`~arma_modlist_tools.fetcher.build_server_index`. + :returns: Report dict:: + + { + "generated_at": "2026-04-07T12:00:00+00:00", + "total_mods": 80, + "on_server": 2, + "missing": 78, + "missing_mods": [ + { + "name": "CBA_A3", + "steam_id": "450814997", + "url": "https://steamcommunity.com/...", + "group": "shared" + }, + ... + ] + } + """ + by_steam_id: dict = server_index.get("by_steam_id", {}) + by_name: dict = server_index.get("by_name", {}) + + from .fetcher import _normalize_name # reuse existing helper + + def _on_server(mod: dict) -> bool: + if mod.get("steam_id") and mod["steam_id"] in by_steam_id: + return True + return _normalize_name(mod.get("name", "")) in by_name + + # Flatten all mods with their group label + all_mods: list[tuple[dict, str]] = [] + for mod in comparison["shared"]["mods"]: + all_mods.append((mod, "shared")) + for preset_name, data in comparison["unique"].items(): + for mod in data["mods"]: + all_mods.append((mod, preset_name)) + + missing_mods = [] + on_server_count = 0 + + for mod, group in all_mods: + if _on_server(mod): + on_server_count += 1 + else: + missing_mods.append({ + "name": mod["name"], + "steam_id": mod.get("steam_id"), + "url": mod.get("url"), + "group": group, + }) + + return { + "generated_at": datetime.now(timezone.utc).isoformat(), + "total_mods": len(all_mods), + "on_server": on_server_count, + "missing": len(missing_mods), + "missing_mods": missing_mods, + } + + +def save_missing_report(report: dict, path: Path) -> None: + """Write *report* as indented JSON to *path*, creating parent dirs as needed.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") diff --git a/check_deps.py b/check_deps.py new file mode 100644 index 0000000..50e81b4 --- /dev/null +++ b/check_deps.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +Verify Python version, OS, and required packages before running the toolchain. + +Usage: + python check_deps.py +""" + +import importlib.metadata +import sys + +# Must not import arma_modlist_tools here — that's what we're checking FOR. + +MIN_PYTHON = (3, 11) +REQUIRED_PACKAGES = ["requests", "tqdm"] + + +def _python_ok() -> bool: + return sys.version_info >= MIN_PYTHON + + +def _get_os_label() -> str: + """Inline OS detection — avoids importing compat before deps are confirmed.""" + import platform + if sys.platform == "win32": + if "Server" in platform.version() or "Server" in platform.uname().version: + return "Windows Server" + return "Windows" + if sys.platform == "linux": + try: + with open("/etc/os-release", encoding="utf-8") as f: + text = f.read().lower() + if "ubuntu" in text: + import os + headless = not (os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")) + return "Ubuntu Server" if headless else "Ubuntu" + except OSError: + pass + return "Linux" + return "Unknown" + + +def _pkg_version(name: str) -> str | None: + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return None + + +def main() -> None: + print() + + # Python version + ver = sys.version.split()[0] + py_ok = _python_ok() + status = "OK" if py_ok else f"NEED >= {MIN_PYTHON[0]}.{MIN_PYTHON[1]}" + print(f" {'Python':<14} {ver:<12} {status}") + + # OS + os_label = _get_os_label() + print(f" {'OS':<14} {os_label}") + print() + + # Packages + missing = [] + for pkg in REQUIRED_PACKAGES: + v = _pkg_version(pkg) + if v: + print(f" {pkg:<14} {v:<12} OK") + else: + print(f" {pkg:<14} {'---':<12} MISSING") + missing.append(pkg) + + print() + + if not py_ok: + print(f" Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]}+ is required.") + sys.exit(1) + + if missing: + print(f" {len(missing)} package(s) missing. Install with:") + print(f" pip install {' '.join(missing)}") + print() + print(" Run check_deps.py again to verify.") + sys.exit(1) + + print(" All checks passed. Ready to run.") + print() + + +if __name__ == "__main__": + main() diff --git a/compare_modlists.py b/compare_modlists.py new file mode 100644 index 0000000..3ab5ee6 --- /dev/null +++ b/compare_modlists.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""CLI entry point: compare all mod presets in modlist_html/ -> modlist_json/comparison.json.""" + +import json + +from arma_modlist_tools import parse_modlist_dir, compare_presets +from arma_modlist_tools.config import load_config + + +def main(): + cfg = load_config() + + presets = parse_modlist_dir(cfg.modlist_html) + if len(presets) < 2: + print("Need at least 2 preset files in modlist_html/ to compare.") + return + + result = compare_presets(*presets) + + cfg.modlist_json.mkdir(exist_ok=True) + cfg.comparison.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") + + shared = result["shared"]["mod_count"] + print(f"Compared: {', '.join(result['compared_presets'])}") + print(f" Shared mods : {shared}") + for preset_name, data in result["unique"].items(): + print(f" Unique to {preset_name}: {data['mod_count']}") + print(f" -> {cfg.comparison}") + + +if __name__ == "__main__": + main() diff --git a/config.template.json b/config.template.json new file mode 100644 index 0000000..5aaadef --- /dev/null +++ b/config.template.json @@ -0,0 +1,13 @@ +{ + "server": { + "base_url": "https://your-caddy-server/arma3mods/", + "username": "your_username", + "password": "your_password" + }, + "paths": { + "arma_dir": "C:\\Path\\To\\Arma 3 Server", + "downloads": "downloads", + "modlist_html": "modlist_html", + "modlist_json": "modlist_json" + } +} diff --git a/fetch_mods.py b/fetch_mods.py new file mode 100644 index 0000000..703a0e2 --- /dev/null +++ b/fetch_mods.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +""" +CLI entry point: download mods from the Caddy file server using comparison.json. + +Folder layout produced: + downloads/ + shared/ <- mods common to all presets + @ace/ + @cba_a3/ + 150th_MW_2026_v1.0/ <- preset-unique mods + @rhsusaf/ + 150th_WW2_2026_V1.0/ + @ifa3_aio/ +""" + +import json +import sys + +from tqdm import tqdm + +from arma_modlist_tools.compat import fix_console_encoding +from arma_modlist_tools.config import load_config +from arma_modlist_tools.fetcher import ( + make_session, + build_server_index, + find_mod_folder, + list_mod_files, + download_file, +) +from arma_modlist_tools.reporter import build_missing_report, save_missing_report + +fix_console_encoding() + + +def _fmt_bytes(n: int) -> str: + for unit in ("B", "KB", "MB", "GB"): + if n < 1024: + return f"{n:.1f} {unit}" + n /= 1024 + return f"{n:.1f} TB" + + +def _build_download_queue(comparison: dict) -> list[tuple[dict, str]]: + """Return flat list of (mod_entry, group_name) for every mod.""" + queue = [] + for mod in comparison["shared"]["mods"]: + queue.append((mod, "shared")) + for preset_name, data in comparison["unique"].items(): + for mod in data["mods"]: + queue.append((mod, preset_name)) + return queue + + +def main() -> None: + cfg = load_config() + + # ---- Load comparison ---- + if not cfg.comparison.exists(): + print(f"ERROR: {cfg.comparison} not found. Run compare_modlists.py first.") + sys.exit(1) + comparison = json.loads(cfg.comparison.read_text(encoding="utf-8")) + queue = _build_download_queue(comparison) + + print(f"Loaded comparison: {', '.join(comparison['compared_presets'])}") + print(f"Total mods to consider: {len(queue)}\n") + + # ---- Build server index ---- + print("Building server index (fetching meta.cpp for each server folder)...") + index = build_server_index(cfg.server_url, cfg.server_auth) + tqdm.write(f" Indexed {len(index['by_steam_id'])} mods by steam_id, " + f"{len(index['by_name'])} by name\n") + + # ---- Resolve mods against server ---- + print("Resolving mods against server index...") + session = make_session(cfg.server_auth) + resolved: list[tuple[dict, str, str]] = [] + not_found: list[dict] = [] + + for mod, group in queue: + url = find_mod_folder(mod, index) + if url: + resolved.append((mod, group, url)) + else: + not_found.append(mod) + + tqdm.write(f" {len(resolved)} matched " + f"{len(not_found)} not found on server\n") + + # ---- Save missing report ---- + report = build_missing_report(comparison, index) + save_missing_report(report, cfg.missing_report) + if not_found: + tqdm.write(f" Missing report saved: {cfg.missing_report} " + f"({report['missing']} mods)\n") + + if not resolved: + print("Nothing to download.") + sys.exit(0) + + # ---- Pre-scan file lists ---- + print("Scanning server for file lists...") + mod_file_lists: list[tuple[dict, str, str, list]] = [] + conflicts: list[str] = [] + + with tqdm(total=len(resolved), unit="mod", desc=" Scanning", leave=False) as bar: + for mod, group, folder_url in resolved: + folder_name = folder_url.rstrip("/").split("/")[-1] + dest_path = cfg.downloads / group / folder_name + files = list_mod_files(folder_url, session) + mod_file_lists.append((mod, group, folder_url, files)) + if dest_path.exists(): + conflicts.append(str(dest_path)) + bar.update(1) + + total_bytes = sum(size for _, _, _, files in mod_file_lists for _, _, size in files) + tqdm.write(f" Total download size: {_fmt_bytes(total_bytes)}\n") + + # ---- Conflict resolution (ask once) ---- + overwrite = False + if conflicts: + print(f"Conflict: {len(conflicts)} destination folder(s) already exist:") + for c in conflicts[:5]: + print(f" {c}") + if len(conflicts) > 5: + print(f" ... and {len(conflicts) - 5} more") + while True: + choice = input("\nOverwrite existing? [s]kip / [o]verwrite: ").strip().lower() + if choice in ("s", "skip"): + break + if choice in ("o", "overwrite"): + overwrite = True + break + print(" Please enter 's' or 'o'.") + print() + + # ---- Download ---- + print(f"Starting downloads: {len(mod_file_lists)} mods\n") + + total_downloaded_bytes = total_downloaded_files = total_skipped_files = 0 + + with tqdm( + total=len(mod_file_lists), + unit="mod", + desc="Overall", + position=0, + dynamic_ncols=True, + ) as mod_bar: + for i, (mod, group, folder_url, files) in enumerate(mod_file_lists, 1): + folder_name = folder_url.rstrip("/").split("/")[-1] + dest_path = cfg.downloads / group / folder_name + + tqdm.write( + f"\n[{i}/{len(mod_file_lists)}] {folder_name}" + f" -> {dest_path}/" + f" (group: {group})" + ) + + mod_bytes = mod_files_dl = mod_files_skip = 0 + + for rel, file_url, size in files: + dest_file = dest_path / rel + if dest_file.exists() and not overwrite: + tqdm.write(f" SKIP {rel}") + mod_files_skip += 1 + continue + + with tqdm( + total=size if size else None, + unit="B", unit_scale=True, unit_divisor=1024, + desc=f" {rel[-45:]:45s}", + position=1, leave=False, dynamic_ncols=True, + ) as file_bar: + n = download_file( + file_url, dest_file, session, + on_chunk=lambda b: file_bar.update(b), + ) + + mod_bytes += n + mod_files_dl += 1 + + total_downloaded_bytes += mod_bytes + total_downloaded_files += mod_files_dl + total_skipped_files += mod_files_skip + + tqdm.write( + f" Done {mod_files_dl} downloaded" + + (f" {mod_files_skip} skipped" if mod_files_skip else "") + + f" {_fmt_bytes(mod_bytes)}" + ) + mod_bar.update(1) + + # ---- Summary ---- + print(f"\n{'─' * 50}") + print(f" Mods processed : {len(mod_file_lists)}") + print(f" Files downloaded: {total_downloaded_files}") + print(f" Files skipped : {total_skipped_files}") + print(f" Mods not found : {len(not_found)}") + print(f" Total downloaded: {_fmt_bytes(total_downloaded_bytes)}") + print(f" Output root : {cfg.downloads.resolve()}") + if not_found: + print(f" Missing report : {cfg.missing_report}") + + +if __name__ == "__main__": + main() diff --git a/link_mods.py b/link_mods.py new file mode 100644 index 0000000..887ef5f --- /dev/null +++ b/link_mods.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +""" +CLI entry point: manage Arma 3 Server junction/symlink links for downloaded mods. + +Usage: + python link_mods.py status --group shared + python link_mods.py link --group shared + python link_mods.py unlink --group shared + +Omit --group to list available groups. +""" + +import argparse +import sys + +from arma_modlist_tools.compat import fix_console_encoding +from arma_modlist_tools.config import load_config +from arma_modlist_tools.linker import get_link_status, link_group, unlink_group, create_junction, remove_junction + +fix_console_encoding() + + +def _available_groups(cfg) -> list[str]: + if not cfg.downloads.is_dir(): + return [] + return sorted(p.name for p in cfg.downloads.iterdir() if p.is_dir()) + + +def _print_separator(width: int = 60) -> None: + print(" " + "-" * width) + + +# --------------------------------------------------------------------------- +# Subcommands +# --------------------------------------------------------------------------- + +def cmd_status(cfg, group: str) -> None: + group_dir = cfg.downloads / group + if not group_dir.is_dir(): + print(f"ERROR: group folder not found: {group_dir}") + sys.exit(1) + + status = get_link_status(group_dir, cfg.arma_dir) + + print() + print(f" Group : {group}") + print(f" Path : {group_dir}") + print(f" Arma : {cfg.arma_dir}") + print() + print(f" {'Mod':<50} Status") + _print_separator() + + for s in status: + icon = "[LINKED]" if s["is_linked"] else "[------]" + print(f" {s['name']:<50} {icon}") + + linked = sum(1 for s in status if s["is_linked"]) + print() + print(f" {linked} / {len(status)} linked") + print() + + +def cmd_link(cfg, group: str) -> None: + group_dir = cfg.downloads / group + if not group_dir.is_dir(): + print(f"ERROR: group folder not found: {group_dir}") + sys.exit(1) + + status = get_link_status(group_dir, cfg.arma_dir) + if not status: + print(f"No @Mod folders found in {group_dir}") + sys.exit(0) + + print() + print(f" Linking group: {group} -> {cfg.arma_dir}") + print() + + linked = already_linked = failed = 0 + + for s in status: + if s["is_linked"]: + print(f" [=] {s['name']:<48} already linked") + already_linked += 1 + continue + if s["link_path"].exists(): + print(f" [!] {s['name']:<48} SKIP — path exists, not a junction") + failed += 1 + continue + ok = create_junction(s["link_path"], s["source_path"]) + if ok: + print(f" [+] {s['name']:<48} linked") + linked += 1 + else: + print(f" [X] {s['name']:<48} FAILED") + failed += 1 + + print() + print(f" Done: {linked} linked, {already_linked} already linked, {failed} failed") + print() + + +def cmd_unlink(cfg, group: str) -> None: + group_dir = cfg.downloads / group + if not group_dir.is_dir(): + print(f"ERROR: group folder not found: {group_dir}") + sys.exit(1) + + status = get_link_status(group_dir, cfg.arma_dir) + linked_count = sum(1 for s in status if s["is_linked"]) + + if linked_count == 0: + print(f"\n Nothing to unlink — 0 active links in group '{group}'.\n") + sys.exit(0) + + print() + print(f" WARNING: This will remove {linked_count} link(s) from Arma 3 Server:") + print(f" {cfg.arma_dir}") + print(f" Group: {group} ({linked_count} linked mod(s))") + print() + + choice = input(" Continue? [y/N]: ").strip().lower() + if choice not in ("y", "yes"): + print(" Aborted.\n") + sys.exit(0) + + print() + + unlinked = not_linked = failed = 0 + + for s in status: + if not s["is_linked"]: + print(f" [=] {s['name']:<48} not linked") + not_linked += 1 + continue + ok, err = remove_junction(s["link_path"]) + if ok: + print(f" [-] {s['name']:<48} unlinked") + unlinked += 1 + else: + print(f" [X] {s['name']:<48} FAILED: {err}") + failed += 1 + + print() + print(f" Done: {unlinked} removed, {not_linked} not linked" + + (f", {failed} failed" if failed else "")) + print() + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def main() -> None: + cfg = load_config() + + parser = argparse.ArgumentParser( + description="Manage Arma 3 Server junction/symlink links for downloaded mods." + ) + parser.add_argument("command", choices=["status", "link", "unlink"]) + parser.add_argument("--group", "-g", metavar="GROUP") + args = parser.parse_args() + + if not args.group: + groups = _available_groups(cfg) + if groups: + print(f"\nAvailable groups in '{cfg.downloads}':") + for g in groups: + print(f" {g}") + print(f"\nUsage: python link_mods.py {args.command} --group \n") + else: + print(f"No groups found in '{cfg.downloads}'. Run fetch_mods.py first.") + sys.exit(0) + + if args.command == "status": + cmd_status(cfg, args.group) + elif args.command == "link": + cmd_link(cfg, args.group) + elif args.command == "unlink": + cmd_unlink(cfg, args.group) + + +if __name__ == "__main__": + main() diff --git a/modlist_html/150th_MW_2026_v1.0.html b/modlist_html/150th_MW_2026_v1.0.html new file mode 100644 index 0000000..7f856ad --- /dev/null +++ b/modlist_html/150th_MW_2026_v1.0.html @@ -0,0 +1,634 @@ + + + + + + + Arma 3 + + + + +

Arma 3 Mods

+

+ To import this preset, drag this file onto the Launcher window. Or click the MODS tab, then PRESET in the top right, then IMPORT at the bottom, and finally select this file. +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
[ANDIA] - FUBAR System (DEV) + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3650593763 +
A3 Thermal Improvement + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2041057379 +
ace + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=463939057 +
ACE 3 Extension (Animations and Actions) + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=766491311 +
Advanced Rappelling + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=713709341 +
Advanced Sling Loading + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=615007497 +
Advanced Towing + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=639837898 +
Al Salman 2.0 + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2857846877 +
Ambient Modules + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2816705133 +
Animated Grenade Throwing + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2935338016 +
Archie, Summer + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3620961988 +
Archie, Winter + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3640984328 +
BackpackOnChest + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=820924072 +
Better Inventory + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2791403093 +
BettIR NVG + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2260572637 +
Breach - Rewrite + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3283645995 +
CBA_A3 + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=450814997 +
CES Resupply + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2744348250 +
Cold War Factions + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2966202074 +
Crows Electronic Warfare + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2515887728 +
Crows Zeus Additions + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2447965207 +
CUP Terrains - Core + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=583496184 +
CUP Terrains - Maps + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=583544987 +
CUP Terrains - Maps 2.0 + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1981964169 +
Devourerking's Necroplague Mutants + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2616555444 +
DUI - Squad Radar + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1638341685 +
Enhanced GPS + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2480263219 +
Enhanced Map + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2467589125 +
Enhanced Movement + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=333310405 +
Improved Melee System + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2291129343 +
JCA - Infantry Arsenal + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3333302397 +
JCA - Infantry Equipment + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3473383676 +
JCA - QOL Essentials V2 + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3032405142 +
NIArms All in One - RHS Compatibility. + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1400574293 +
NIArms All In One (V14 Onwards) + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2595680138 +
NIArms All in One- ACE Compatibility + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1400566170 +
NIArms for Unconventional Actors + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2915856399 +
Novogorsk + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2979021411 +
Pook Boat Pack + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1529074643 +
Realistic Ragdoll Physics + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3639557777 +
RHS Additions - Rewrite + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3553838240 +
RHSAFRF + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=843425103 +
RHSGREF + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=843593391 +
RHSSAF + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=843632231 +
RHSUSAF + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=843577117 +
Root's Anomalies - Zeus Module + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2882374586 +
Saint Kapaulio + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=939686262 +
Some Effects Rework: Impacts + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3596299237 +
Speshal Core + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3283642267 +
Task Force Arrowhead Radio (BETA!!!) + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=894678801 +
Unconventional Actors + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2890312862 +
W28 + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3205870510 +
WBK Immersive Animations + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3165450999 +
Weather Plus + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2735613231 +
WebKnight Flashlights and Headlamps + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2572487482 +
WebKnight's Zombies and Creatures + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2789152015 +
Zeus Enhanced + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1779063631 +
Zeus Immersion Sounds + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2461386136 +
Zombies and Demons + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=501966277 +
Zombies and Demons ACE integration + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1606871585 +
+
+
+ Created by Arma 3 Launcher by Bohemia Interactive. +
+ + \ No newline at end of file diff --git a/modlist_html/150th_WW2_2026_V1.0.html b/modlist_html/150th_WW2_2026_V1.0.html new file mode 100644 index 0000000..451c9fc --- /dev/null +++ b/modlist_html/150th_WW2_2026_V1.0.html @@ -0,0 +1,463 @@ + + + + + + + Arma 3 + + + + +

Arma 3 Mods

+

+ To import this preset, drag this file onto the Launcher window. Or click the MODS tab, then PRESET in the top right, then IMPORT at the bottom, and finally select this file. +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
[ANDIA] - FUBAR System (DEV) + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3650593763 +
[SWU] Immersion Sound Pack + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=946763963 +
150th Languard Additional Ace Flags + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3467298844 +
ace + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=463939057 +
ACE 3 Extension (Animations and Actions) + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=766491311 +
Ambient Modules + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2816705133 +
Animated Grenade Throwing + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2935338016 +
BackpackOnChest + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=820924072 +
CBA_A3 + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=450814997 +
Crows Zeus Additions + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2447965207 +
CUP Terrains - Core + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=583496184 +
Darkest December 1944 + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3373660050 +
DUI - Squad Radar + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1638341685 +
Eifel Forest 1944 + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3445102949 +
Enhanced Movement + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=333310405 +
Enhanced Movement Rework + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2034363662 +
Flying Legends + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2012417505 +
Groesbeek Heights 1944 + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3633127181 +
Hurtgen Forest + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3214719358 +
IFA3 AIO + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2648308937 +
Improved Melee System + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2291129343 +
JM's Second Assault + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3493205282 +
M24 Chaffee + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3301951691 +
Market Garden Grave Bridge + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3524533280 +
MRPH Infantry Charge + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3560796660 +
Pegasus Bridge + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3391363624 +
Realistic Ragdoll Physics + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3639557777 +
Saint Pierre Du Mont + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3494072989 +
Secret Weapons Reloaded + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2710902874 +
Some Effects Rework: Impacts + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3596299237 +
Spearhead 1944 - IFA3 SPE Tank Overhaul + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3305447657 +
Spearhead 1944 - Secret Weapons Compatibility + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=3014048725 +
Task Force Arrowhead Radio (BETA!!!) + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=894678801 +
US GEAr: Units (IFA3) + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1496363537 +
US General Equipment and Accessories + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1399447232 +
WebKnight's Zombies and Creatures + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2789152015 +
Zeus Enhanced + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1779063631 +
ZEUS WARGAME [RTS mod] + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=2932697000 +
ZluskeN Whistle and Bugle Mod + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=884372152 +
Zombies and Demons + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=501966277 +
Zombies and Demons ACE integration + Steam + + https://steamcommunity.com/sharedfiles/filedetails/?id=1606871585 +
+
+
+ Created by Arma 3 Launcher by Bohemia Interactive. +
+ + \ No newline at end of file diff --git a/parse_modlist.py b/parse_modlist.py new file mode 100644 index 0000000..f886846 --- /dev/null +++ b/parse_modlist.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""CLI entry point: parse all mod preset HTMLs in modlist_html/ -> modlist_json/.""" + +import json + +from arma_modlist_tools import parse_modlist_dir +from arma_modlist_tools.config import load_config + + +def main(): + cfg = load_config() + cfg.modlist_json.mkdir(exist_ok=True) + + presets = parse_modlist_dir(cfg.modlist_html) + if not presets: + print(f"No .html files found in {cfg.modlist_html}/") + return + + for preset in presets: + out_file = cfg.modlist_json / (preset["preset_name"] + ".json") + out_file.write_text(json.dumps(preset, indent=2, ensure_ascii=False), encoding="utf-8") + print(f"{preset['source_file']} -> {out_file} ({preset['mod_count']} mods)") + + +if __name__ == "__main__": + main() diff --git a/report_missing.py b/report_missing.py new file mode 100644 index 0000000..860b20b --- /dev/null +++ b/report_missing.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +CLI entry point: cross-reference comparison.json against the file server and +report which required mods are missing. + +Usage: + python report_missing.py +""" + +import json +import sys + +from arma_modlist_tools.compat import fix_console_encoding +from arma_modlist_tools.config import load_config +from arma_modlist_tools.fetcher import build_server_index +from arma_modlist_tools.reporter import build_missing_report, save_missing_report + +fix_console_encoding() + + +def main() -> None: + cfg = load_config() + + if not cfg.comparison.exists(): + print(f"ERROR: {cfg.comparison} not found. Run compare_modlists.py first.") + sys.exit(1) + + comparison = json.loads(cfg.comparison.read_text(encoding="utf-8")) + total_mods = ( + comparison["shared"]["mod_count"] + + sum(d["mod_count"] for d in comparison["unique"].values()) + ) + + # ---- Build server index ---- + print(f"\nChecking server index...") + index = build_server_index(cfg.server_url, cfg.server_auth) + print(f" {len(index['by_steam_id'])} mods indexed\n") + + # ---- Build report ---- + print(f"Cross-referencing {total_mods} required mods...") + report = build_missing_report(comparison, index) + save_missing_report(report, cfg.missing_report) + + # ---- Print table ---- + missing = report["missing_mods"] + if missing: + print(f"\n Missing from server ({len(missing)}):\n") + print(f" {'steam_id':<14} {'Group':<28} Name") + print(f" {'-'*14} {'-'*28} {'-'*40}") + for m in missing: + sid = m["steam_id"] or "---" + group = m["group"] + print(f" {sid:<14} {group:<28} {m['name']}") + else: + print("\n All required mods are on the server.") + + print() + print(f" {report['on_server']} / {report['total_mods']} found on server") + print(f" Report saved: {cfg.missing_report}") + print() + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5bb8c66 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +tqdm diff --git a/run.py b/run.py new file mode 100644 index 0000000..60096f6 --- /dev/null +++ b/run.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +""" +Orchestrator: run the full pipeline in sequence. + +Steps: + 1. parse — parse HTML presets -> modlist_json/ + 2. compare — compare presets -> comparison.json + 3. fetch — download mods from server (also saves missing_report.json) + 4. link — create junctions/symlinks to Arma 3 Server + +Usage: + python run.py # full pipeline, all groups + python run.py --skip-fetch --skip-link # parse + compare only + python run.py --skip-parse --skip-compare --skip-fetch --group shared +""" + +import argparse +import json +import sys + +from tqdm import tqdm + +from arma_modlist_tools.compat import fix_console_encoding +from arma_modlist_tools.config import load_config +from arma_modlist_tools.parser import parse_modlist_dir +from arma_modlist_tools.compare import compare_presets +from arma_modlist_tools.fetcher import ( + build_server_index, find_mod_folder, + list_mod_files, download_file, make_session, +) +from arma_modlist_tools.linker import link_group +from arma_modlist_tools.reporter import build_missing_report, save_missing_report + +fix_console_encoding() + + +def _fmt_bytes(n: int) -> str: + for unit in ("B", "KB", "MB", "GB"): + if n < 1024: + return f"{n:.1f} {unit}" + n /= 1024 + return f"{n:.1f} TB" + + +def _header(step: int, total: int, name: str) -> None: + print(f"\n{'='*56}") + print(f" Step {step}/{total}: {name}") + print(f"{'='*56}\n") + + +# --------------------------------------------------------------------------- +# Step implementations +# --------------------------------------------------------------------------- + +def step_parse(cfg) -> None: + cfg.modlist_json.mkdir(exist_ok=True) + presets = parse_modlist_dir(cfg.modlist_html) + if not presets: + print(f" No .html files found in {cfg.modlist_html}/") + return + for preset in presets: + out = cfg.modlist_json / (preset["preset_name"] + ".json") + out.write_text(json.dumps(preset, indent=2, ensure_ascii=False), encoding="utf-8") + print(f" {preset['source_file']} -> {out} ({preset['mod_count']} mods)") + + +def step_compare(cfg) -> None: + presets = parse_modlist_dir(cfg.modlist_html) + if len(presets) < 2: + print(" Need at least 2 preset files to compare.") + return + result = compare_presets(*presets) + cfg.modlist_json.mkdir(exist_ok=True) + cfg.comparison.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") + print(f" Compared: {', '.join(result['compared_presets'])}") + print(f" Shared: {result['shared']['mod_count']} | ", end="") + print(" ".join(f"{k}: {v['mod_count']} unique" for k, v in result["unique"].items())) + print(f" -> {cfg.comparison}") + + +def step_fetch(cfg) -> None: + if not cfg.comparison.exists(): + print(f" ERROR: {cfg.comparison} not found. Run parse + compare first.") + return + + comparison = json.loads(cfg.comparison.read_text(encoding="utf-8")) + queue: list[tuple[dict, str]] = [] + for mod in comparison["shared"]["mods"]: + queue.append((mod, "shared")) + for preset_name, data in comparison["unique"].items(): + for mod in data["mods"]: + queue.append((mod, preset_name)) + + print(f" Building server index...") + index = build_server_index(cfg.server_url, cfg.server_auth) + print(f" Indexed {len(index['by_steam_id'])} mods\n") + + session = make_session(cfg.server_auth) + resolved = [] + for mod, group in queue: + url = find_mod_folder(mod, index) + if url: + resolved.append((mod, group, url)) + + # Save missing report + report = build_missing_report(comparison, index) + save_missing_report(report, cfg.missing_report) + print(f" {len(resolved)} / {len(queue)} mods found on server") + if report["missing"]: + print(f" {report['missing']} missing — saved to {cfg.missing_report}") + + if not resolved: + print(" Nothing to download.") + return + + # Scan + download + print() + mod_file_lists = [] + for mod, group, folder_url in resolved: + folder_name = folder_url.rstrip("/").split("/")[-1] + dest_path = cfg.downloads / group / folder_name + files = list_mod_files(folder_url, session) + mod_file_lists.append((mod, group, folder_url, dest_path, files)) + + total_bytes = 0 + with tqdm(total=len(mod_file_lists), unit="mod", desc=" Fetching", position=0, dynamic_ncols=True) as mod_bar: + for mod, group, folder_url, dest_path, files in mod_file_lists: + folder_name = folder_url.rstrip("/").split("/")[-1] + mod_bytes = 0 + for rel, file_url, size in files: + dest_file = dest_path / rel + if dest_file.exists(): + continue + with tqdm( + total=size if size else None, + unit="B", unit_scale=True, unit_divisor=1024, + desc=f" {rel[-40:]:40s}", + position=1, leave=False, dynamic_ncols=True, + ) as file_bar: + n = download_file(file_url, dest_file, session, + on_chunk=lambda b: file_bar.update(b)) + mod_bytes += n + total_bytes += mod_bytes + mod_bar.update(1) + + print(f"\n Done. {len(mod_file_lists)} mods {_fmt_bytes(total_bytes)}") + + +def step_link(cfg, groups: list[str]) -> None: + if not cfg.arma_dir.exists(): + print(f" NOTE: Arma dir not found ({cfg.arma_dir}) — skipping link step.") + return + + for group in groups: + group_dir = cfg.downloads / group + if not group_dir.is_dir(): + print(f" SKIP {group} — folder not found") + continue + result = link_group(group_dir, cfg.arma_dir) + print(f" {group:<32} " + f"{result['linked']} linked, " + f"{result['already_linked']} already linked" + + (f", {result['failed']} failed" if result["failed"] else "")) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def main() -> None: + cfg = load_config() + + parser = argparse.ArgumentParser(description="Run the full mod management pipeline.") + parser.add_argument("--skip-parse", action="store_true") + parser.add_argument("--skip-compare", action="store_true") + parser.add_argument("--skip-fetch", action="store_true") + parser.add_argument("--skip-link", action="store_true") + parser.add_argument("--group", "-g", metavar="GROUP", + help="Link only this group (default: all groups in downloads/)") + args = parser.parse_args() + + steps = [ + (not args.skip_parse, "Parse presets"), + (not args.skip_compare, "Compare presets"), + (not args.skip_fetch, "Fetch mods"), + (not args.skip_link, "Link mods"), + ] + active_count = sum(1 for run, _ in steps if run) + step_num = 0 + + if args.skip_parse and args.skip_compare and args.skip_fetch and args.skip_link: + print("All steps skipped — nothing to do.") + sys.exit(0) + + for (run, name), fn_args in zip(steps, [ + (cfg,), + (cfg,), + (cfg,), + None, # handled separately + ]): + if not run: + continue + step_num += 1 + _header(step_num, active_count, name) + + if name == "Link mods": + if args.group: + groups = [args.group] + else: + groups = sorted( + p.name for p in cfg.downloads.iterdir() + if cfg.downloads.is_dir() and p.is_dir() + ) if cfg.downloads.is_dir() else [] + step_link(cfg, groups) + elif name == "Parse presets": + step_parse(cfg) + elif name == "Compare presets": + step_compare(cfg) + elif name == "Fetch mods": + step_fetch(cfg) + + print(f"\n{'='*56}") + print(" Pipeline complete.") + print(f"{'='*56}\n") + + +if __name__ == "__main__": + main() diff --git a/sync_missing.py b/sync_missing.py new file mode 100644 index 0000000..f9847f8 --- /dev/null +++ b/sync_missing.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +CLI entry point: fetch mods that were previously missing but have since been +added to the file server, then link them. + +Flow: + 1. Load missing_report.json + 2. Re-check server index for newly available mods + 3. Download newly available mods to their correct group folder + 4. Update missing_report.json (remove the ones now downloaded) + 5. Run link_group for each affected group + +Usage: + python sync_missing.py +""" + +import json +import sys + +from tqdm import tqdm + +from arma_modlist_tools.compat import fix_console_encoding +from arma_modlist_tools.config import load_config +from arma_modlist_tools.fetcher import ( + build_server_index, find_mod_folder, + list_mod_files, download_file, make_session, +) +from arma_modlist_tools.linker import link_group +from arma_modlist_tools.reporter import save_missing_report + +fix_console_encoding() + + +def _fmt_bytes(n: int) -> str: + for unit in ("B", "KB", "MB", "GB"): + if n < 1024: + return f"{n:.1f} {unit}" + n /= 1024 + return f"{n:.1f} TB" + + +def main() -> None: + cfg = load_config() + + # ---- Load missing report ---- + if not cfg.missing_report.exists(): + print(f"\nERROR: {cfg.missing_report} not found.") + print("Run report_missing.py or fetch_mods.py first.\n") + sys.exit(1) + + report = json.loads(cfg.missing_report.read_text(encoding="utf-8")) + previously_missing = report["missing_mods"] + + if not previously_missing: + print("\nNo mods in missing report — nothing to sync.\n") + sys.exit(0) + + print(f"\nLoading missing report: {len(previously_missing)} mods previously missing") + + # ---- Re-check server index ---- + print("Re-checking server index...") + index = build_server_index(cfg.server_url, cfg.server_auth) + print(f" {len(index['by_steam_id'])} mods indexed\n") + + session = make_session(cfg.server_auth) + + # ---- Resolve which are now available ---- + now_available: list[tuple[dict, str]] = [] # (mod, folder_url) + still_missing: list[dict] = [] + + for mod in previously_missing: + url = find_mod_folder(mod, index) + if url: + now_available.append((mod, url)) + else: + still_missing.append(mod) + + print(f" Newly available: {len(now_available)} mods") + print(f" Still missing : {len(still_missing)} mods\n") + + if not now_available: + print("No new mods available on server yet.\n") + sys.exit(0) + + # ---- Download newly available mods ---- + affected_groups: set[str] = set() + total_bytes = 0 + + with tqdm(total=len(now_available), unit="mod", desc="Downloading", position=0, dynamic_ncols=True) as mod_bar: + for mod, folder_url in now_available: + group = mod["group"] + folder_name = folder_url.rstrip("/").split("/")[-1] + dest_path = cfg.downloads / group / folder_name + + tqdm.write(f" [+] {folder_name:<48} -> downloads/{group}/") + + files = list_mod_files(folder_url, session) + mod_bytes = 0 + + for rel, file_url, size in files: + dest_file = dest_path / rel + if dest_file.exists(): + continue + with tqdm( + total=size if size else None, + unit="B", unit_scale=True, unit_divisor=1024, + desc=f" {rel[-45:]:45s}", + position=1, leave=False, dynamic_ncols=True, + ) as file_bar: + n = download_file( + file_url, dest_file, session, + on_chunk=lambda b: file_bar.update(b), + ) + mod_bytes += n + + tqdm.write(f" Done {_fmt_bytes(mod_bytes)}") + total_bytes += mod_bytes + affected_groups.add(group) + mod_bar.update(1) + + print(f"\n Downloaded: {len(now_available)} mods {_fmt_bytes(total_bytes)}") + + # ---- Update missing report ---- + report["missing_mods"] = still_missing + report["missing"] = len(still_missing) + report["on_server"] = report["total_mods"] - len(still_missing) + save_missing_report(report, cfg.missing_report) + print(f" Missing report updated: {len(still_missing)} still missing\n") + + # ---- Link newly downloaded mods ---- + if not cfg.arma_dir.exists(): + print(f" NOTE: Arma dir not found ({cfg.arma_dir}) — skipping link step.") + print(" Run link_mods.py manually when ready.\n") + sys.exit(0) + + print("Linking newly added mods...") + for group in sorted(affected_groups): + group_dir = cfg.downloads / group + result = link_group(group_dir, cfg.arma_dir) + print(f" {group:<30} " + f"{result['linked']} new linked, " + f"{result['already_linked']} already linked" + + (f", {result['failed']} failed" if result["failed"] else "")) + + print("\nDone.\n") + + +if __name__ == "__main__": + main() diff --git a/test_suite.py b/test_suite.py new file mode 100644 index 0000000..4402fc5 --- /dev/null +++ b/test_suite.py @@ -0,0 +1,1145 @@ +#!/usr/bin/env python3 +""" +Comprehensive test suite for rev-arma-modlist-tools. + +Run: + python test_suite.py + +Tests are grouped by module. Network tests (fetcher) are skipped if the server +is unreachable. Linker tests use temp directories so they never touch real paths. +""" + +import json +import os +import sys +import tempfile +import textwrap +import traceback +from pathlib import Path + +# --------------------------------------------------------------------------- +# Test harness (no external dependencies) +# --------------------------------------------------------------------------- + +_passed = _failed = _skipped = 0 +_current_group = "" + + +def group(name: str) -> None: + global _current_group + _current_group = name + print(f"\n{'-'*60}") + print(f" {name}") + print(f"{'-'*60}") + + +def test(name: str, fn) -> None: + global _passed, _failed + try: + fn() + print(f" [PASS] {name}") + _passed += 1 + except Exception as exc: + print(f" [FAIL] {name}") + for line in traceback.format_exc().splitlines(): + print(f" {line}") + _failed += 1 + + +def skip(name: str, reason: str) -> None: + global _skipped + print(f" [SKIP] {name} ({reason})") + _skipped += 1 + + +def assert_eq(a, b, msg=""): + assert a == b, f"{msg}\n expected: {b!r}\n got: {a!r}" + + +def assert_in(item, container, msg=""): + assert item in container, f"{msg}\n {item!r} not in {container!r}" + + +def assert_raises(exc_type, fn, *args, **kwargs): + try: + fn(*args, **kwargs) + except exc_type: + return + raise AssertionError(f"Expected {exc_type.__name__} but no exception was raised") + + +# --------------------------------------------------------------------------- +# 1. compat +# --------------------------------------------------------------------------- + +group("compat") + +from arma_modlist_tools.compat import ( + is_windows, is_linux, get_os_label, fix_console_encoding, +) + + +def _test_is_windows_linux_mutually_exclusive(): + # Exactly one of is_windows()/is_linux() can be True; both can be False (Unknown) + assert not (is_windows() and is_linux()), "Cannot be both Windows and Linux" + + +def _test_os_label_is_string(): + label = get_os_label() + assert isinstance(label, str) and label, "OS label must be non-empty string" + valid = {"Windows", "Windows Server", "Ubuntu", "Ubuntu Server", "Linux", "Unknown"} + assert label in valid, f"Unexpected OS label: {label!r}" + + +def _test_fix_console_encoding_idempotent(): + # Calling it twice must not raise + fix_console_encoding() + fix_console_encoding() + + +def _test_is_windows_matches_platform(): + import platform + assert is_windows() == (sys.platform == "win32") + + +def _test_is_linux_matches_platform(): + assert is_linux() == (sys.platform == "linux") + + +def _test_os_label_consistent_with_platform(): + label = get_os_label() + if is_windows(): + assert "Windows" in label + elif is_linux(): + assert label in {"Ubuntu", "Ubuntu Server", "Linux"} + else: + assert label == "Unknown" + + +test("is_windows and is_linux are mutually exclusive", _test_is_windows_linux_mutually_exclusive) +test("get_os_label returns a valid label", _test_os_label_is_string) +test("fix_console_encoding is idempotent", _test_fix_console_encoding_idempotent) +test("is_windows() matches sys.platform", _test_is_windows_matches_platform) +test("is_linux() matches sys.platform", _test_is_linux_matches_platform) +test("OS label is consistent with platform", _test_os_label_consistent_with_platform) + + +# --------------------------------------------------------------------------- +# 2. config +# --------------------------------------------------------------------------- + +group("config") + +from arma_modlist_tools.config import load_config, Config + +_VALID_CONFIG = { + "server": { + "base_url": "https://example.com/mods/", + "username": "user", + "password": "pass", + }, + "paths": { + "arma_dir": "C:\\arma3", + "downloads": "downloads", + "modlist_html": "modlist_html", + "modlist_json": "modlist_json", + }, +} + + +def _test_load_config_from_explicit_path(): + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(_VALID_CONFIG, f) + tmp = f.name + try: + cfg = load_config(tmp) + assert_eq(cfg.server_url, "https://example.com/mods/") + assert_eq(cfg.server_auth, ("user", "pass")) + assert_eq(cfg.arma_dir, Path("C:\\arma3")) + assert_eq(cfg.downloads, Path("downloads")) + assert_eq(cfg.modlist_html, Path("modlist_html")) + assert_eq(cfg.modlist_json, Path("modlist_json")) + assert_eq(cfg.comparison, Path("modlist_json/comparison.json")) + assert_eq(cfg.missing_report, Path("modlist_json/missing_report.json")) + finally: + os.unlink(tmp) + + +def _test_load_config_file_not_found(): + assert_raises(FileNotFoundError, load_config, "nonexistent_1234567890.json") + + +def _test_load_config_missing_key_raises(): + bad = {"server": {"base_url": "x"}} # missing paths + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(bad, f) + tmp = f.name + try: + assert_raises(KeyError, load_config, tmp) + finally: + os.unlink(tmp) + + +def _test_config_derived_paths(): + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(_VALID_CONFIG, f) + tmp = f.name + try: + cfg = load_config(tmp) + # comparison must live inside modlist_json + assert cfg.comparison.parent == cfg.modlist_json + assert cfg.comparison.name == "comparison.json" + # missing_report must live inside modlist_json + assert cfg.missing_report.parent == cfg.modlist_json + assert cfg.missing_report.name == "missing_report.json" + finally: + os.unlink(tmp) + + +def _test_load_config_from_cwd(): + """load_config() without args should find the real config.json at CWD.""" + original_cwd = Path.cwd() + project_root = Path(__file__).parent + os.chdir(project_root) + try: + cfg = load_config() + assert cfg.server_url, "server_url must not be empty" + finally: + os.chdir(original_cwd) + + +test("load_config with explicit path", _test_load_config_from_explicit_path) +test("load_config raises FileNotFoundError for missing file", _test_load_config_file_not_found) +test("load_config raises KeyError for incomplete config", _test_load_config_missing_key_raises) +test("Config derived paths (comparison, missing_report)", _test_config_derived_paths) +test("load_config() auto-discovers config.json from CWD", _test_load_config_from_cwd) + + +# --------------------------------------------------------------------------- +# 3. parser +# --------------------------------------------------------------------------- + +group("parser") + +from arma_modlist_tools.parser import ( + parse_mod_entry, parse_modlist_html, parse_modlist_dir, +) +import xml.etree.ElementTree as ET + +_VALID_TR_XML = textwrap.dedent("""\ + + CBA_A3 + + + + + +""") + +_LOCAL_TR_XML = textwrap.dedent("""\ + + My Local Mod + + + + +""") + +_NO_NAME_TR_XML = textwrap.dedent("""\ + + + + + +""") + +_MINIMAL_HTML = textwrap.dedent("""\ + + + Test + + + + + + + + + + + + + + +
ACE3 + +
CBA_A3 + +
LocalMod
+ + +""") + + +def _test_parse_mod_entry_steam(): + tr = ET.fromstring(_VALID_TR_XML) + entry = parse_mod_entry(tr) + assert entry is not None + assert_eq(entry["name"], "CBA_A3") + assert_eq(entry["source"], "steam") + assert_eq(entry["steam_id"], "450814997") + assert "450814997" in entry["url"] + + +def _test_parse_mod_entry_local(): + tr = ET.fromstring(_LOCAL_TR_XML) + entry = parse_mod_entry(tr) + assert entry is not None + assert_eq(entry["name"], "My Local Mod") + assert_eq(entry["source"], "local") + assert entry["steam_id"] is None + assert entry["url"] is None + + +def _test_parse_mod_entry_no_name_returns_none(): + tr = ET.fromstring(_NO_NAME_TR_XML) + entry = parse_mod_entry(tr) + assert entry is None, "Entry without DisplayName must return None" + + +def _test_parse_modlist_html_from_file(): + with tempfile.NamedTemporaryFile( + mode="w", suffix=".html", delete=False, encoding="utf-8" + ) as f: + f.write(_MINIMAL_HTML) + tmp = Path(f.name) + try: + preset = parse_modlist_html(tmp) + assert_eq(preset["source_file"], tmp.name) + assert_eq(preset["mod_count"], 3) + assert len(preset["mods"]) == 3 + names = [m["name"] for m in preset["mods"]] + assert_in("ACE3", names) + assert_in("CBA_A3", names) + assert_in("LocalMod", names) + # steam IDs + steam_ids = {m["steam_id"] for m in preset["mods"]} + assert_in("463939057", steam_ids) + assert_in("450814997", steam_ids) + assert_in(None, steam_ids) # LocalMod has no steam_id + finally: + tmp.unlink() + + +def _test_parse_modlist_html_not_found(): + assert_raises(FileNotFoundError, parse_modlist_html, "no_such_file_xyz.html") + + +def _test_parse_modlist_dir_real_files(): + d = Path(__file__).parent / "modlist_html" + if not d.is_dir(): + raise AssertionError(f"modlist_html directory not found: {d}") + presets = parse_modlist_dir(d) + assert len(presets) >= 2, f"Expected >=2 presets, got {len(presets)}" + for p in presets: + assert p["mod_count"] > 0, f"{p['source_file']} has 0 mods" + assert len(p["mods"]) == p["mod_count"] + + +def _test_parse_modlist_dir_empty(): + with tempfile.TemporaryDirectory() as d: + result = parse_modlist_dir(d) + assert_eq(result, []) + + +def _test_parse_modlist_dir_not_a_dir(): + assert_raises(NotADirectoryError, parse_modlist_dir, "no_such_directory_xyz") + + +def _test_parse_modlist_dir_returns_sorted(): + with tempfile.TemporaryDirectory() as d: + d = Path(d) + for name in ["c_preset.html", "a_preset.html", "b_preset.html"]: + f = d / name + f.write_text(_MINIMAL_HTML, encoding="utf-8") + presets = parse_modlist_dir(d) + names = [p["source_file"] for p in presets] + assert names == sorted(names), "parse_modlist_dir must return results sorted by filename" + + +test("parse_mod_entry: Steam mod", _test_parse_mod_entry_steam) +test("parse_mod_entry: Local mod", _test_parse_mod_entry_local) +test("parse_mod_entry: No name returns None", _test_parse_mod_entry_no_name_returns_none) +test("parse_modlist_html: from temp HTML file", _test_parse_modlist_html_from_file) +test("parse_modlist_html: FileNotFoundError", _test_parse_modlist_html_not_found) +test("parse_modlist_dir: real modlist_html/ files", _test_parse_modlist_dir_real_files) +test("parse_modlist_dir: empty directory", _test_parse_modlist_dir_empty) +test("parse_modlist_dir: non-existent directory", _test_parse_modlist_dir_not_a_dir) +test("parse_modlist_dir: results are sorted by filename", _test_parse_modlist_dir_returns_sorted) + + +# --------------------------------------------------------------------------- +# 4. compare +# --------------------------------------------------------------------------- + +group("compare") + +from arma_modlist_tools.compare import compare_presets + +_P1 = { + "preset_name": "Preset_A", + "source_file": "preset_a.html", + "mod_count": 3, + "mods": [ + {"name": "CBA_A3", "source": "steam", "url": None, "steam_id": "450814997"}, + {"name": "ACE3", "source": "steam", "url": None, "steam_id": "463939057"}, + {"name": "OnlyInA", "source": "steam", "url": None, "steam_id": "111111111"}, + ], +} + +_P2 = { + "preset_name": "Preset_B", + "source_file": "preset_b.html", + "mod_count": 3, + "mods": [ + {"name": "CBA_A3", "source": "steam", "url": None, "steam_id": "450814997"}, + {"name": "ACE3", "source": "steam", "url": None, "steam_id": "463939057"}, + {"name": "OnlyInB", "source": "steam", "url": None, "steam_id": "222222222"}, + ], +} + +_P_LOCAL = { + "preset_name": "Preset_Local", + "source_file": "preset_local.html", + "mod_count": 2, + "mods": [ + {"name": "CBA_A3", "source": "steam", "url": None, "steam_id": "450814997"}, + {"name": "MyLocalMod", "source": "local", "url": None, "steam_id": None}, + ], +} + +_P_IDENTICAL = { + "preset_name": "Preset_C", + "source_file": "preset_c.html", + "mod_count": 3, + "mods": [ + {"name": "CBA_A3", "source": "steam", "url": None, "steam_id": "450814997"}, + {"name": "ACE3", "source": "steam", "url": None, "steam_id": "463939057"}, + {"name": "OnlyInA", "source": "steam", "url": None, "steam_id": "111111111"}, + ], +} + + +def _test_compare_basic_shared_and_unique(): + result = compare_presets(_P1, _P2) + assert_eq(sorted(result["compared_presets"]), ["Preset_A", "Preset_B"]) + shared_ids = {m["steam_id"] for m in result["shared"]["mods"]} + assert_eq(shared_ids, {"450814997", "463939057"}) + assert_eq(result["shared"]["mod_count"], 2) + unique_a = {m["steam_id"] for m in result["unique"]["Preset_A"]["mods"]} + unique_b = {m["steam_id"] for m in result["unique"]["Preset_B"]["mods"]} + assert_eq(unique_a, {"111111111"}) + assert_eq(unique_b, {"222222222"}) + + +def _test_compare_requires_two_or_more(): + assert_raises(ValueError, compare_presets, _P1) + + +def _test_compare_three_presets(): + result = compare_presets(_P1, _P2, _P_LOCAL) + # Only CBA_A3 (450814997) is in all three + shared_ids = {m["steam_id"] for m in result["shared"]["mods"]} + assert_eq(shared_ids, {"450814997"}) + assert "Preset_Local" in result["unique"] + + +def _test_compare_identical_presets_all_shared(): + result = compare_presets(_P1, _P_IDENTICAL) + assert_eq(result["shared"]["mod_count"], 3) + for data in result["unique"].values(): + assert_eq(data["mod_count"], 0) + + +def _test_compare_no_shared_mods(): + p_no_overlap = { + "preset_name": "Preset_X", + "source_file": "px.html", + "mod_count": 1, + "mods": [{"name": "Exclusive", "source": "steam", "url": None, "steam_id": "999999999"}], + } + result = compare_presets(_P1, p_no_overlap) + assert_eq(result["shared"]["mod_count"], 0) + assert_eq(len(result["shared"]["mods"]), 0) + + +def _test_compare_local_mod_uses_name_as_key(): + # Two presets share a local mod by name (no steam_id) + p_a = { + "preset_name": "A", + "source_file": "a.html", + "mod_count": 1, + "mods": [{"name": "MyLocalMod", "source": "local", "url": None, "steam_id": None}], + } + p_b = { + "preset_name": "B", + "source_file": "b.html", + "mod_count": 1, + "mods": [{"name": "MyLocalMod", "source": "local", "url": None, "steam_id": None}], + } + result = compare_presets(p_a, p_b) + assert_eq(result["shared"]["mod_count"], 1) + assert_eq(result["shared"]["mods"][0]["name"], "MyLocalMod") + + +def _test_compare_preserves_all_fields(): + result = compare_presets(_P1, _P2) + for mod in result["shared"]["mods"]: + assert "name" in mod + assert "source" in mod + assert "steam_id" in mod + + +def _test_compare_result_counts_consistent(): + result = compare_presets(_P1, _P2) + for preset in (_P1, _P2): + pname = preset["preset_name"] + total = result["shared"]["mod_count"] + result["unique"][pname]["mod_count"] + assert_eq(total, preset["mod_count"], + f"shared + unique must equal total for {pname}") + + +test("compare_presets: shared and unique breakdown", _test_compare_basic_shared_and_unique) +test("compare_presets: requires >= 2 presets", _test_compare_requires_two_or_more) +test("compare_presets: three presets, intersection only", _test_compare_three_presets) +test("compare_presets: identical presets → all shared", _test_compare_identical_presets_all_shared) +test("compare_presets: no overlap → shared is empty", _test_compare_no_shared_mods) +test("compare_presets: local mod matched by name", _test_compare_local_mod_uses_name_as_key) +test("compare_presets: result mods retain all fields", _test_compare_preserves_all_fields) +test("compare_presets: shared + unique = total per preset", _test_compare_result_counts_consistent) + + +# --------------------------------------------------------------------------- +# 5. fetcher (pure functions only — no network) +# --------------------------------------------------------------------------- + +group("fetcher (pure functions, no network)") + +from arma_modlist_tools.fetcher import ( + _normalize_name, _parse_meta_cpp, _folder_url, find_mod_folder, +) + + +def _test_normalize_name_strips_at(): + assert_eq(_normalize_name("@ace"), "ace") + assert_eq(_normalize_name("@CBA_A3"), "cbaa3") + + +def _test_normalize_name_lowercase(): + assert_eq(_normalize_name("@RHS_USAF"), "rhsusaf") + + +def _test_normalize_name_removes_spaces_and_special(): + assert_eq(_normalize_name("@My Mod (v2)"), "mymodv2") + + +def _test_normalize_name_no_at(): + assert_eq(_normalize_name("ace"), "ace") + + +def _test_parse_meta_cpp_standard(): + txt = 'publishedid = 463939057;\nname = "ACE3";' + assert_eq(_parse_meta_cpp(txt), "463939057") + + +def _test_parse_meta_cpp_no_spaces(): + assert_eq(_parse_meta_cpp("publishedid=123456;"), "123456") + + +def _test_parse_meta_cpp_case_insensitive(): + assert_eq(_parse_meta_cpp("PublishedId = 999;"), "999") + + +def _test_parse_meta_cpp_missing(): + assert _parse_meta_cpp("name = somemod;") is None + + +def _test_folder_url_trailing_slash(): + url = _folder_url("https://example.com/mods", "@ace") + assert url.endswith("/"), "folder_url must end with /" + assert "@ace" in url + + +def _test_find_mod_folder_by_steam_id(): + index = { + "by_steam_id": {"450814997": "https://example.com/mods/@cba_a3/"}, + "by_name": {}, + } + mod = {"steam_id": "450814997", "name": "CBA_A3"} + url = find_mod_folder(mod, index) + assert_eq(url, "https://example.com/mods/@cba_a3/") + + +def _test_find_mod_folder_by_name_fallback(): + index = { + "by_steam_id": {}, + "by_name": {"ace3": "https://example.com/mods/@ace3/"}, + } + mod = {"steam_id": None, "name": "ACE3"} + url = find_mod_folder(mod, index) + assert_eq(url, "https://example.com/mods/@ace3/") + + +def _test_find_mod_folder_steam_id_preferred_over_name(): + index = { + "by_steam_id": {"111": "https://example.com/mods/@right/"}, + "by_name": {"mymod": "https://example.com/mods/@wrong/"}, + } + mod = {"steam_id": "111", "name": "MyMod"} + url = find_mod_folder(mod, index) + assert_eq(url, "https://example.com/mods/@right/") + + +def _test_find_mod_folder_not_found(): + index = {"by_steam_id": {}, "by_name": {}} + mod = {"steam_id": "999", "name": "Missing"} + assert find_mod_folder(mod, index) is None + + +def _test_find_mod_folder_normalized_name_match(): + index = { + "by_steam_id": {}, + "by_name": {"rhsusaf": "https://example.com/mods/@rhs_usaf/"}, + } + mod = {"steam_id": None, "name": "@RHS_USAF"} + url = find_mod_folder(mod, index) + assert_eq(url, "https://example.com/mods/@rhs_usaf/") + + +test("_normalize_name: strips leading @", _test_normalize_name_strips_at) +test("_normalize_name: lowercases", _test_normalize_name_lowercase) +test("_normalize_name: removes spaces and special chars", _test_normalize_name_removes_spaces_and_special) +test("_normalize_name: no @ prefix still works", _test_normalize_name_no_at) +test("_parse_meta_cpp: standard format", _test_parse_meta_cpp_standard) +test("_parse_meta_cpp: no spaces", _test_parse_meta_cpp_no_spaces) +test("_parse_meta_cpp: case-insensitive", _test_parse_meta_cpp_case_insensitive) +test("_parse_meta_cpp: missing → None", _test_parse_meta_cpp_missing) +test("_folder_url: always trailing slash", _test_folder_url_trailing_slash) +test("find_mod_folder: by steam_id", _test_find_mod_folder_by_steam_id) +test("find_mod_folder: by name fallback", _test_find_mod_folder_by_name_fallback) +test("find_mod_folder: steam_id preferred over name", _test_find_mod_folder_steam_id_preferred_over_name) +test("find_mod_folder: not found → None", _test_find_mod_folder_not_found) +test("find_mod_folder: normalized @ name match", _test_find_mod_folder_normalized_name_match) + +# ---- list_mod_updates (uses temp dirs, no network) ---- + +from arma_modlist_tools.fetcher import list_mod_updates +import unittest.mock as mock + + +def _test_list_mod_updates_all_uptodate(): + """All local files exist with matching sizes → empty result.""" + with tempfile.TemporaryDirectory() as d: + dest = Path(d) + # Create local files matching server sizes + (dest / "addons").mkdir() + (dest / "addons" / "mod.pbo").write_bytes(b"x" * 1000) + server_files = [("addons/mod.pbo", "https://x.com/addons/mod.pbo", 1000)] + with mock.patch("arma_modlist_tools.fetcher.list_mod_files", return_value=server_files): + result = list_mod_updates("https://x.com/", dest, None) + assert_eq(result, [], "No stale files expected when sizes match") + + +def _test_list_mod_updates_missing_file(): + """Local file does not exist → included.""" + with tempfile.TemporaryDirectory() as d: + dest = Path(d) + server_files = [("addons/new.pbo", "https://x.com/addons/new.pbo", 500)] + with mock.patch("arma_modlist_tools.fetcher.list_mod_files", return_value=server_files): + result = list_mod_updates("https://x.com/", dest, None) + assert_eq(len(result), 1) + assert_eq(result[0][0], "addons/new.pbo") + + +def _test_list_mod_updates_size_mismatch(): + """Local file exists but size differs → included.""" + with tempfile.TemporaryDirectory() as d: + dest = Path(d) + (dest / "addons").mkdir() + (dest / "addons" / "mod.pbo").write_bytes(b"x" * 999) # local: 999 bytes + server_files = [("addons/mod.pbo", "https://x.com/addons/mod.pbo", 1000)] # server: 1000 + with mock.patch("arma_modlist_tools.fetcher.list_mod_files", return_value=server_files): + result = list_mod_updates("https://x.com/", dest, None) + assert_eq(len(result), 1) + assert_eq(result[0][2], 1000, "Returned entry must carry server size") + + +def _test_list_mod_updates_zero_server_size_skips_check(): + """Server reports size=0 (unknown) — only include if file is missing.""" + with tempfile.TemporaryDirectory() as d: + dest = Path(d) + (dest / "addons").mkdir() + (dest / "addons" / "mod.pbo").write_bytes(b"x" * 500) # exists locally + server_files = [("addons/mod.pbo", "https://x.com/addons/mod.pbo", 0)] + with mock.patch("arma_modlist_tools.fetcher.list_mod_files", return_value=server_files): + result = list_mod_updates("https://x.com/", dest, None) + # size=0 → skip size check, file exists → not stale + assert_eq(result, [], "Existing file with server size=0 must not be re-downloaded") + + +def _test_list_mod_updates_mixed(): + """One up-to-date, one missing, one stale → two results.""" + with tempfile.TemporaryDirectory() as d: + dest = Path(d) + (dest / "addons").mkdir() + (dest / "addons" / "current.pbo").write_bytes(b"x" * 100) # matches server + (dest / "addons" / "stale.pbo").write_bytes(b"x" * 50) # differs from server + # missing.pbo does not exist + server_files = [ + ("addons/current.pbo", "https://x.com/addons/current.pbo", 100), + ("addons/stale.pbo", "https://x.com/addons/stale.pbo", 200), + ("addons/missing.pbo", "https://x.com/addons/missing.pbo", 300), + ] + with mock.patch("arma_modlist_tools.fetcher.list_mod_files", return_value=server_files): + result = list_mod_updates("https://x.com/", dest, None) + result_names = {r[0] for r in result} + assert_eq(result_names, {"addons/stale.pbo", "addons/missing.pbo"}) + assert "addons/current.pbo" not in result_names + + +test("list_mod_updates: all files up-to-date → empty", _test_list_mod_updates_all_uptodate) +test("list_mod_updates: missing file → included", _test_list_mod_updates_missing_file) +test("list_mod_updates: size mismatch → included", _test_list_mod_updates_size_mismatch) +test("list_mod_updates: server size=0 → skip size check", _test_list_mod_updates_zero_server_size_skips_check) +test("list_mod_updates: mixed state (up-to-date, stale, missing)", _test_list_mod_updates_mixed) + + +# --------------------------------------------------------------------------- +# 6. reporter +# --------------------------------------------------------------------------- + +group("reporter") + +from arma_modlist_tools.reporter import build_missing_report, save_missing_report + +_COMPARISON = { + "compared_presets": ["Preset_A", "Preset_B"], + "shared": { + "mod_count": 2, + "mods": [ + {"name": "CBA_A3", "steam_id": "450814997", "url": None, "source": "steam"}, + {"name": "ACE3", "steam_id": "463939057", "url": None, "source": "steam"}, + ], + }, + "unique": { + "Preset_A": { + "mod_count": 1, + "mods": [{"name": "OnlyA", "steam_id": "111", "url": None, "source": "steam"}], + }, + "Preset_B": { + "mod_count": 1, + "mods": [{"name": "OnlyB", "steam_id": "222", "url": None, "source": "steam"}], + }, + }, +} + +_INDEX_ALL = { + "by_steam_id": { + "450814997": "https://x.com/@cba/", + "463939057": "https://x.com/@ace/", + "111": "https://x.com/@onlya/", + "222": "https://x.com/@onlyb/", + }, + "by_name": {}, +} + +_INDEX_NONE: dict = {"by_steam_id": {}, "by_name": {}} + +_INDEX_PARTIAL = { + "by_steam_id": { + "450814997": "https://x.com/@cba/", + "463939057": "https://x.com/@ace/", + }, + "by_name": {}, +} + + +def _test_build_missing_report_all_found(): + report = build_missing_report(_COMPARISON, _INDEX_ALL) + assert_eq(report["total_mods"], 4) + assert_eq(report["on_server"], 4) + assert_eq(report["missing"], 0) + assert_eq(report["missing_mods"], []) + + +def _test_build_missing_report_all_missing(): + report = build_missing_report(_COMPARISON, _INDEX_NONE) + assert_eq(report["total_mods"], 4) + assert_eq(report["on_server"], 0) + assert_eq(report["missing"], 4) + assert_eq(len(report["missing_mods"]), 4) + + +def _test_build_missing_report_partial(): + report = build_missing_report(_COMPARISON, _INDEX_PARTIAL) + assert_eq(report["on_server"], 2) + assert_eq(report["missing"], 2) + missing_ids = {m["steam_id"] for m in report["missing_mods"]} + assert_eq(missing_ids, {"111", "222"}) + + +def _test_build_missing_report_group_field(): + report = build_missing_report(_COMPARISON, _INDEX_NONE) + groups = {m["group"] for m in report["missing_mods"]} + assert_in("shared", groups) + assert_in("Preset_A", groups) + assert_in("Preset_B", groups) + + +def _test_build_missing_report_shared_group_label(): + report = build_missing_report(_COMPARISON, _INDEX_NONE) + shared_entries = [m for m in report["missing_mods"] if m["name"] in ("CBA_A3", "ACE3")] + for e in shared_entries: + assert_eq(e["group"], "shared", f"{e['name']} must have group='shared'") + + +def _test_build_missing_report_has_timestamp(): + report = build_missing_report(_COMPARISON, _INDEX_ALL) + assert "generated_at" in report + assert isinstance(report["generated_at"], str) + assert "T" in report["generated_at"] # ISO 8601 format + + +def _test_save_and_reload_missing_report(): + report = build_missing_report(_COMPARISON, _INDEX_PARTIAL) + with tempfile.TemporaryDirectory() as d: + out = Path(d) / "sub" / "missing_report.json" + save_missing_report(report, out) + assert out.exists(), "save_missing_report must create the file" + loaded = json.loads(out.read_text(encoding="utf-8")) + assert_eq(loaded["missing"], report["missing"]) + assert_eq(loaded["total_mods"], report["total_mods"]) + + +def _test_save_missing_report_creates_parent_dirs(): + with tempfile.TemporaryDirectory() as d: + deep = Path(d) / "a" / "b" / "c" / "report.json" + save_missing_report({"test": True}, deep) + assert deep.exists() + + +test("build_missing_report: all mods found", _test_build_missing_report_all_found) +test("build_missing_report: all mods missing", _test_build_missing_report_all_missing) +test("build_missing_report: partial coverage", _test_build_missing_report_partial) +test("build_missing_report: group field present on all entries", _test_build_missing_report_group_field) +test("build_missing_report: shared mods labeled group='shared'", _test_build_missing_report_shared_group_label) +test("build_missing_report: includes ISO timestamp", _test_build_missing_report_has_timestamp) +test("save_missing_report: write and reload roundtrip", _test_save_and_reload_missing_report) +test("save_missing_report: creates parent directories", _test_save_missing_report_creates_parent_dirs) + + +# --------------------------------------------------------------------------- +# 7. linker +# --------------------------------------------------------------------------- + +group("linker") + +from arma_modlist_tools.linker import ( + get_mod_folders, get_link_status, + create_junction, remove_junction, + link_group, unlink_group, + _is_junction, +) + + +def _make_fake_mods(base: Path, names: list[str]) -> None: + """Create @ModName directories with a dummy file inside each.""" + for name in names: + mod_dir = base / name + mod_dir.mkdir(parents=True) + (mod_dir / "mod.cpp").write_text(f"// {name}", encoding="utf-8") + + +def _test_get_mod_folders_finds_at_dirs(): + with tempfile.TemporaryDirectory() as d: + d = Path(d) + _make_fake_mods(d, ["@ace", "@cba_a3", "@rhs"]) + (d / "not_a_mod").mkdir() # should be ignored + (d / "readme.txt").write_text("x") # should be ignored + folders = get_mod_folders(d) + names = [f.name for f in folders] + assert_eq(sorted(names), ["@ace", "@cba_a3", "@rhs"]) + assert "not_a_mod" not in names + + +def _test_get_mod_folders_empty_dir(): + with tempfile.TemporaryDirectory() as d: + folders = get_mod_folders(Path(d)) + assert_eq(folders, []) + + +def _test_get_mod_folders_nonexistent(): + result = get_mod_folders(Path("no_such_dir_xyz_abc")) + assert_eq(result, []) + + +def _test_get_link_status_unlinked(): + with tempfile.TemporaryDirectory() as src_d, tempfile.TemporaryDirectory() as arma_d: + src_d = Path(src_d) + arma_d = Path(arma_d) + _make_fake_mods(src_d, ["@ace", "@cba_a3"]) + status = get_link_status(src_d, arma_d) + assert_eq(len(status), 2) + for s in status: + assert not s["is_linked"], f"{s['name']} should not be linked yet" + assert s["link_path"].parent == arma_d + assert s["source_path"].exists() + + +def _test_create_and_detect_junction(): + with tempfile.TemporaryDirectory() as src_d, tempfile.TemporaryDirectory() as arma_d: + src_d = Path(src_d) + arma_d = Path(arma_d) + _make_fake_mods(src_d, ["@mymod"]) + target = src_d / "@mymod" + link = arma_d / "@mymod" + ok = create_junction(link, target) + assert ok, "create_junction must return True on success" + assert link.exists(), "link must exist after creation" + assert _is_junction(link), "_is_junction must detect the new junction" + # Verify junction points to the right files + assert (link / "mod.cpp").exists(), "linked junction must expose target contents" + + +def _test_remove_junction(): + with tempfile.TemporaryDirectory() as src_d, tempfile.TemporaryDirectory() as arma_d: + src_d = Path(src_d) + arma_d = Path(arma_d) + _make_fake_mods(src_d, ["@mymod"]) + target = src_d / "@mymod" + link = arma_d / "@mymod" + create_junction(link, target) + assert _is_junction(link) + ok, err = remove_junction(link) + assert ok, f"remove_junction failed: {err}" + assert not link.exists(), "junction must be gone after removal" + # Target must be untouched + assert target.exists(), "target directory must survive junction removal" + assert (target / "mod.cpp").exists(), "target files must survive junction removal" + + +def _test_link_group_full_flow(): + with tempfile.TemporaryDirectory() as src_d, tempfile.TemporaryDirectory() as arma_d: + src_d = Path(src_d) + arma_d = Path(arma_d) + _make_fake_mods(src_d, ["@ace", "@cba_a3", "@rhs"]) + result = link_group(src_d, arma_d) + assert_eq(result["linked"], 3) + assert_eq(result["already_linked"], 0) + assert_eq(result["failed"], 0) + # All links exist + for name in ["@ace", "@cba_a3", "@rhs"]: + assert _is_junction(arma_d / name), f"{name} must be linked" + + +def _test_link_group_idempotent(): + with tempfile.TemporaryDirectory() as src_d, tempfile.TemporaryDirectory() as arma_d: + src_d = Path(src_d) + arma_d = Path(arma_d) + _make_fake_mods(src_d, ["@ace"]) + link_group(src_d, arma_d) + result2 = link_group(src_d, arma_d) + assert_eq(result2["linked"], 0) + assert_eq(result2["already_linked"], 1) + assert_eq(result2["failed"], 0) + + +def _test_unlink_group_removes_links(): + with tempfile.TemporaryDirectory() as src_d, tempfile.TemporaryDirectory() as arma_d: + src_d = Path(src_d) + arma_d = Path(arma_d) + _make_fake_mods(src_d, ["@ace", "@cba_a3"]) + link_group(src_d, arma_d) + result = unlink_group(src_d, arma_d) + assert_eq(result["unlinked"], 2) + assert_eq(result["not_linked"], 0) + assert_eq(result["failed"], 0) + # Links gone, sources still there + for name in ["@ace", "@cba_a3"]: + assert not (arma_d / name).exists(), f"Link {name} must be gone" + assert (src_d / name).exists(), f"Source {name} must survive" + + +def _test_unlink_group_nothing_linked(): + with tempfile.TemporaryDirectory() as src_d, tempfile.TemporaryDirectory() as arma_d: + src_d = Path(src_d) + arma_d = Path(arma_d) + _make_fake_mods(src_d, ["@ace"]) + result = unlink_group(src_d, arma_d) + assert_eq(result["unlinked"], 0) + assert_eq(result["not_linked"], 1) + + +def _test_get_link_status_after_linking(): + with tempfile.TemporaryDirectory() as src_d, tempfile.TemporaryDirectory() as arma_d: + src_d = Path(src_d) + arma_d = Path(arma_d) + _make_fake_mods(src_d, ["@ace"]) + link_group(src_d, arma_d) + status = get_link_status(src_d, arma_d) + assert_eq(len(status), 1) + assert status[0]["is_linked"], "status must reflect active junction" + + +def _test_link_group_skips_existing_non_junction(): + """If arma_dir already has a regular folder with the same name, must not overwrite.""" + with tempfile.TemporaryDirectory() as src_d, tempfile.TemporaryDirectory() as arma_d: + src_d = Path(src_d) + arma_d = Path(arma_d) + _make_fake_mods(src_d, ["@ace"]) + # Create a plain (non-junction) directory at the link path + (arma_d / "@ace").mkdir() + result = link_group(src_d, arma_d) + assert_eq(result["failed"], 1, "Must report failure for path that exists but is not a junction") + assert_eq(result["linked"], 0) + + +test("get_mod_folders: finds @-prefixed dirs only", _test_get_mod_folders_finds_at_dirs) +test("get_mod_folders: empty directory", _test_get_mod_folders_empty_dir) +test("get_mod_folders: nonexistent path → []", _test_get_mod_folders_nonexistent) +test("get_link_status: all unlinked initially", _test_get_link_status_unlinked) +test("create_junction + _is_junction detection", _test_create_and_detect_junction) +test("remove_junction: removes link, target untouched", _test_remove_junction) +test("link_group: full link flow", _test_link_group_full_flow) +test("link_group: idempotent (already_linked on second call)", _test_link_group_idempotent) +test("unlink_group: removes all links", _test_unlink_group_removes_links) +test("unlink_group: nothing linked → not_linked count", _test_unlink_group_nothing_linked) +test("get_link_status: reflects active junctions", _test_get_link_status_after_linking) +test("link_group: skips plain dir (fails safely)", _test_link_group_skips_existing_non_junction) + + +# --------------------------------------------------------------------------- +# 8. __init__ exports +# --------------------------------------------------------------------------- + +group("__init__ public exports") + +import arma_modlist_tools as pkg + +_EXPECTED_EXPORTS = [ + "parse_mod_entry", "parse_modlist_html", "parse_modlist_dir", + "compare_presets", + "make_session", "build_server_index", "find_mod_folder", + "list_mod_files", "list_mod_updates", "download_file", "download_mod_folder", + "get_mod_folders", "get_link_status", + "create_junction", "remove_junction", + "link_group", "unlink_group", + "load_config", "Config", + "is_windows", "is_linux", "get_os_label", "fix_console_encoding", + "build_missing_report", "save_missing_report", +] + + +def _test_all_exports_present(): + missing = [name for name in _EXPECTED_EXPORTS if not hasattr(pkg, name)] + assert not missing, f"Missing exports from arma_modlist_tools: {missing}" + + +def _test_exports_are_callable(): + non_callable = [ + name for name in _EXPECTED_EXPORTS + if hasattr(pkg, name) and not callable(getattr(pkg, name)) + and not isinstance(getattr(pkg, name), type) + ] + assert not non_callable, f"Non-callable exports: {non_callable}" + + +test("all expected symbols exported", _test_all_exports_present) +test("all exported symbols are callable or types", _test_exports_are_callable) + + +# --------------------------------------------------------------------------- +# 9. Integration: parse → compare → reporter (offline) +# --------------------------------------------------------------------------- + +group("integration: parse → compare → reporter (offline)") + + +def _test_end_to_end_offline(): + """Parse real HTML files, compare them, build a mock report.""" + html_dir = Path(__file__).parent / "modlist_html" + presets = parse_modlist_dir(html_dir) + assert len(presets) >= 2, "Need >=2 HTML presets for integration test" + + comparison = compare_presets(*presets) + total = comparison["shared"]["mod_count"] + sum( + d["mod_count"] for d in comparison["unique"].values() + ) + assert total > 0, "Comparison must have mods" + + # Build report against empty index (all missing) + report_all_missing = build_missing_report(comparison, {"by_steam_id": {}, "by_name": {}}) + assert_eq(report_all_missing["missing"], total) + + # Build report against full index (all found) + full_by_id = { + mod["steam_id"]: f"https://x.com/@mod{i}/" + for i, mod in enumerate( + comparison["shared"]["mods"] + + [m for d in comparison["unique"].values() for m in d["mods"]] + ) + if mod.get("steam_id") + } + full_by_name = { + _normalize_name(mod["name"]): f"https://x.com/@mod{i}/" + for i, mod in enumerate( + comparison["shared"]["mods"] + + [m for d in comparison["unique"].values() for m in d["mods"]] + ) + if not mod.get("steam_id") + } + report_all_found = build_missing_report( + comparison, {"by_steam_id": full_by_id, "by_name": full_by_name} + ) + assert_eq(report_all_found["missing"], 0) + + +def _test_comparison_json_consistent_with_html(): + """The real comparison.json on disk must match a fresh parse+compare.""" + html_dir = Path(__file__).parent / "modlist_html" + json_file = Path(__file__).parent / "modlist_json" / "comparison.json" + if not json_file.exists(): + raise AssertionError(f"comparison.json not found: {json_file}") + + presets = parse_modlist_dir(html_dir) + fresh = compare_presets(*presets) + on_disk = json.loads(json_file.read_text(encoding="utf-8")) + + assert_eq( + sorted(fresh["compared_presets"]), + sorted(on_disk["compared_presets"]), + ) + assert_eq(fresh["shared"]["mod_count"], on_disk["shared"]["mod_count"]) + for pname in fresh["compared_presets"]: + assert_eq( + fresh["unique"][pname]["mod_count"], + on_disk["unique"][pname]["mod_count"], + f"unique mod count mismatch for {pname}", + ) + + +test("end-to-end offline pipeline (parse → compare → report)", _test_end_to_end_offline) +test("comparison.json on disk matches fresh parse+compare", _test_comparison_json_consistent_with_html) + + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- + +total = _passed + _failed + _skipped +print(f"\n{'='*60}") +print(f" Results: {_passed} passed, {_failed} failed, {_skipped} skipped ({total} total)") +print(f"{'='*60}\n") + +if _failed: + sys.exit(1) diff --git a/update_mods.py b/update_mods.py new file mode 100644 index 0000000..d7fa30c --- /dev/null +++ b/update_mods.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +""" +CLI entry point: re-download mod files that have changed on the server. + +Use this after you have updated mod files on the Caddy server without +changing the modlist structure (same mods, same Steam IDs, new file versions). + +Detection method: compare local file sizes against server file sizes. +A file is considered stale when it is missing locally OR its local size +differs from the server-reported size. Use --force to ignore size checks +and re-download every file unconditionally. + +Usage: + python update_mods.py # check all groups/mods + python update_mods.py --group shared # limit to one group + python update_mods.py --mod @ace # limit to one mod folder + python update_mods.py --force # re-download everything + python update_mods.py --force --group shared # force-update one group +""" + +import argparse +import sys + +from tqdm import tqdm + +from arma_modlist_tools.compat import fix_console_encoding +from arma_modlist_tools.config import load_config +from arma_modlist_tools.fetcher import ( + build_server_index, find_mod_folder, + list_mod_files, list_mod_updates, + download_file, make_session, +) + +fix_console_encoding() + + +def _fmt_bytes(n: int) -> str: + for unit in ("B", "KB", "MB", "GB"): + if n < 1024: + return f"{n:.1f} {unit}" + n /= 1024 + return f"{n:.1f} TB" + + +def _collect_targets(cfg, group_filter: str | None, mod_filter: str | None) -> list[tuple[str, str, object]]: + """ + Walk downloads/ and return (group, folder_name, folder_path) for each + @Mod folder that passes the group/mod filters. + """ + targets = [] + if not cfg.downloads.is_dir(): + return targets + for group_dir in sorted(cfg.downloads.iterdir()): + if not group_dir.is_dir(): + continue + if group_filter and group_dir.name != group_filter: + continue + for mod_dir in sorted(group_dir.iterdir()): + if not mod_dir.is_dir() or not mod_dir.name.startswith("@"): + continue + if mod_filter and mod_dir.name != mod_filter: + continue + targets.append((group_dir.name, mod_dir.name, mod_dir)) + return targets + + +def main() -> None: + cfg = load_config() + + parser = argparse.ArgumentParser( + description="Re-download mod files that have changed on the server." + ) + parser.add_argument("--group", "-g", metavar="GROUP", + help="Only check mods in this group folder (e.g. shared)") + parser.add_argument("--mod", "-m", metavar="MOD", + help="Only check this mod folder name (e.g. @ace)") + parser.add_argument("--force", action="store_true", + help="Re-download all files regardless of size match") + args = parser.parse_args() + + # ---- Collect local mod folders to check ---- + targets = _collect_targets(cfg, args.group, args.mod) + if not targets: + print("\nNo mod folders found matching the given filters.\n") + sys.exit(0) + + # ---- Build server index ---- + print(f"\nBuilding server index...") + index = build_server_index(cfg.server_url, cfg.server_auth) + print(f" {len(index['by_steam_id'])} mods indexed\n") + + session = make_session(cfg.server_auth) + + mode = "force" if args.force else "size-check" + print(f" Mode: {mode}") + print(f" Checking {len(targets)} mod folder(s)...\n") + + # Column widths for the summary table + COL_MOD = 44 + COL_GROUP = 24 + + total_checked = total_updated = total_bytes = 0 + not_on_server = [] + + for group, folder_name, mod_dir in targets: + # Find this mod on the server by name (no steam_id available from local dir) + mod_stub = {"name": folder_name, "steam_id": None} + folder_url = find_mod_folder(mod_stub, index) + + if not folder_url: + tqdm.write(f" [?] {folder_name:<{COL_MOD}} {group:<{COL_GROUP}} not found on server") + not_on_server.append(f"{group}/{folder_name}") + continue + + # Determine which files need downloading + if args.force: + stale = list_mod_files(folder_url, session) + else: + stale = list_mod_updates(folder_url, mod_dir, session) + + all_files = list_mod_files(folder_url, session) if not args.force else stale + checked = len(all_files) if not args.force else len(stale) + + if not stale: + print(f" [=] {folder_name:<{COL_MOD}} {group:<{COL_GROUP}} {checked} files up-to-date") + total_checked += checked + continue + + # Download stale files + mod_bytes = 0 + with tqdm( + total=len(stale), unit="file", + desc=f" {folder_name[-COL_MOD:]:<{COL_MOD}}", + position=0, leave=True, dynamic_ncols=True, + ) as file_bar: + for rel, url, size in stale: + dest_file = mod_dir / rel + with tqdm( + total=size if size else None, + unit="B", unit_scale=True, unit_divisor=1024, + desc=f" {rel[-40:]:40s}", + position=1, leave=False, dynamic_ncols=True, + ) as chunk_bar: + n = download_file(url, dest_file, session, + on_chunk=lambda b: chunk_bar.update(b)) + mod_bytes += n + file_bar.update(1) + + total_checked += checked + total_updated += len(stale) + total_bytes += mod_bytes + print(f" [+] {folder_name:<{COL_MOD}} {group:<{COL_GROUP}} " + f"{checked} files {len(stale)} updated ({_fmt_bytes(mod_bytes)})") + + print(f"\n{'='*56}") + print(f" Total: {total_checked} files checked, " + f"{total_updated} updated, " + f"{_fmt_bytes(total_bytes)} downloaded") + if not_on_server: + print(f" Not found on server ({len(not_on_server)}): {', '.join(not_on_server)}") + print(f"{'='*56}\n") + + if total_updated == 0 and not not_on_server: + print(" All mods are up-to-date.\n") + + +if __name__ == "__main__": + main()