diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b00ca8d..54bba73d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,7 +53,7 @@ jobs: ## Installation ```bash - pip install mmdet==${{ steps.get_version.outputs.version }} + pip install visdet==${{ steps.get_version.outputs.version }} ``` ## Documentation diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b7a3efd..a9b902dd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,7 +40,7 @@ jobs: - name: Run tests with coverage if: matrix.python-version == '3.14' - run: uv run pytest --cov=mmdet --cov-report=term-missing --cov-report=xml --ignore tests/test_modal_coco_training_on_modal.py + run: uv run pytest --cov=visdet --cov-report=term-missing --cov-report=xml --ignore tests/test_modal_coco_training_on_modal.py - name: Upload coverage to Codecov if: matrix.python-version == '3.14' diff --git a/docs/yaml_config_system.md b/docs/yaml_config_system.md index 83a6eb7e..f3b0569f 100644 --- a/docs/yaml_config_system.md +++ b/docs/yaml_config_system.md @@ -224,7 +224,7 @@ python tools/convert_config.py \ Run the test suite to verify the YAML system: ```bash -python scripts/test_yaml_simple.py +python scripts/test_yaml_config.py ``` This tests: diff --git a/pyproject.toml b/pyproject.toml index c8281cb7..538bd68c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,7 +144,7 @@ addopts = "-v --tb=short --strict-markers" norecursedirs = ["archive", ".git", ".venv", "build", "dist"] [tool.coverage.run] -source = ["mmdet"] +source = ["visdet"] omit = ["*/tests/*", "*/test_*.py"] [tool.coverage.report] diff --git a/scripts/generate_model_zoo.py b/scripts/generate_model_zoo.py index 6f581f9a..1d2925b3 100644 --- a/scripts/generate_model_zoo.py +++ b/scripts/generate_model_zoo.py @@ -1,5 +1,6 @@ """Generate comprehensive model zoo documentation from config READMEs.""" +import argparse import re from pathlib import Path from typing import Dict, List, Tuple @@ -185,18 +186,17 @@ def generate_model_zoo_page() -> str: [ "## Using Pre-trained Models", "", - "To use any pre-trained model from the model zoo:", + "To train with a preset from the model zoo:", "", "```python", - "from mmdet.apis import init_detector, inference_detector", + "from visdet import SimpleRunner", "", - "# Load model", - "config_file = 'configs/models/faster_rcnn_r50.yaml'", - "checkpoint_file = 'checkpoints/faster_rcnn_r50_fpn_1x_coco.pth'", - "model = init_detector(config_file, checkpoint_file, device='cuda:0')", - "", - "# Run inference", - "result = inference_detector(model, 'demo/demo.jpg')", + "runner = SimpleRunner(", + " model='mask_rcnn_swin_s',", + " dataset='coco_instance_segmentation',", + " epochs=12,", + ")", + "runner.train()", "```", "", "## Training Custom Models", @@ -213,12 +213,25 @@ def generate_model_zoo_page() -> str: return "\n".join(lines) +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate model-zoo documentation page.") + parser.add_argument( + "--output", + default="docs/model-zoo.md", + help="Output markdown path (default: docs/model-zoo.md)", + ) + return parser.parse_args() + + if __name__ == "__main__": + args = parse_args() + # Generate the page content = generate_model_zoo_page() # Write to docs - output_path = Path("mkdocs_docs/model-zoo.md") + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(content) print(f"✓ Generated model zoo documentation: {output_path}") diff --git a/scripts/test_presets_simple.py b/scripts/test_presets_simple.py deleted file mode 100644 index 2e1815d1..00000000 --- a/scripts/test_presets_simple.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python3 -"""Simple test for preset registry (avoids full visdet import). - -This script tests just the preset system without importing -the full visdet package. -""" - -import sys -from pathlib import Path - -# Add visdet to path -repo_root = Path(__file__).parent.parent -sys.path.insert(0, str(repo_root / "visdet/visdet")) - - -def test_preset_discovery(): - """Test preset discovery without full package import.""" - print("=" * 80) - print("Testing Preset Discovery (Lightweight)") - print("=" * 80) - - # Import only the preset registry, not the full package - from presets.registry import ( - DATASET_PRESETS, - MODEL_PRESETS, - OPTIMIZER_PRESETS, - SCHEDULER_PRESETS, - ) - - # Test model presets - print("\n" + "-" * 80) - print("Model Presets:") - print("-" * 80) - models = MODEL_PRESETS.list() - for model in models: - print(f" - {model}") - assert len(models) > 0, "No model presets found" - print(f"✓ Found {len(models)} model presets") - - # Test dataset presets - print("\n" + "-" * 80) - print("Dataset Presets:") - print("-" * 80) - datasets = DATASET_PRESETS.list() - for dataset in datasets: - print(f" - {dataset}") - assert len(datasets) > 0, "No dataset presets found" - print(f"✓ Found {len(datasets)} dataset presets") - - # Test optimizer presets - print("\n" + "-" * 80) - print("Optimizer Presets:") - print("-" * 80) - optimizers = OPTIMIZER_PRESETS.list() - for optimizer in optimizers: - print(f" - {optimizer}") - assert len(optimizers) > 0, "No optimizer presets found" - print(f"✓ Found {len(optimizers)} optimizer presets") - - # Test scheduler presets - print("\n" + "-" * 80) - print("Scheduler Presets:") - print("-" * 80) - schedulers = SCHEDULER_PRESETS.list() - for scheduler in schedulers: - print(f" - {scheduler}") - print(f"✓ Found {len(schedulers)} scheduler presets") - - return True - - -def test_preset_loading(): - """Test that presets can be loaded.""" - print("\n" + "=" * 80) - print("Testing Preset Loading") - print("=" * 80) - - from presets.registry import DATASET_PRESETS, MODEL_PRESETS, OPTIMIZER_PRESETS - - # Test model loading - print("\n" + "-" * 80) - print("Loading: mask_rcnn_swin_s") - print("-" * 80) - try: - model_cfg = MODEL_PRESETS.get("mask_rcnn_swin_s") - assert isinstance(model_cfg, dict) - assert "type" in model_cfg - print(f"✓ Loaded model config (type={model_cfg['type']})") - print(f" - Keys: {', '.join(list(model_cfg.keys())[:10])}") - except Exception as e: - print(f"✗ Failed to load: {e}") - return False - - # Test dataset loading - print("\n" + "-" * 80) - print("Loading: coco_instance_segmentation") - print("-" * 80) - try: - dataset_cfg = DATASET_PRESETS.get("coco_instance_segmentation") - assert isinstance(dataset_cfg, dict) - assert "type" in dataset_cfg - print(f"✓ Loaded dataset config (type={dataset_cfg['type']})") - print(f" - Keys: {', '.join(list(dataset_cfg.keys())[:10])}") - except Exception as e: - print(f"✗ Failed to load: {e}") - return False - - # Test optimizer loading - print("\n" + "-" * 80) - print("Loading: adamw_default") - print("-" * 80) - try: - optimizer_cfg = OPTIMIZER_PRESETS.get("adamw_default") - assert isinstance(optimizer_cfg, dict) - assert "type" in optimizer_cfg - print(f"✓ Loaded optimizer config (type={optimizer_cfg['type']})") - print(f" - lr={optimizer_cfg.get('lr')}, weight_decay={optimizer_cfg.get('weight_decay')}") - except Exception as e: - print(f"✗ Failed to load: {e}") - return False - - return True - - -def test_preset_registration(): - """Test custom preset registration.""" - print("\n" + "=" * 80) - print("Testing Custom Preset Registration") - print("=" * 80) - - from presets.registry import MODEL_PRESETS - - # Register custom preset - print("\n" + "-" * 80) - print("Registering: my_custom_model") - print("-" * 80) - custom_cfg = {"type": "CustomModel", "param1": "value1", "param2": 42} - MODEL_PRESETS.register("my_custom_model", custom_cfg) - - # Verify it appears in list - models = MODEL_PRESETS.list() - assert "my_custom_model" in models - print("✓ Custom preset appears in list") - - # Load it back - loaded_cfg = MODEL_PRESETS.get("my_custom_model") - assert loaded_cfg == custom_cfg - print("✓ Custom preset loaded correctly") - - return True - - -def test_lazy_loading(): - """Test that loading is actually lazy.""" - print("\n" + "=" * 80) - print("Testing Lazy Loading") - print("=" * 80) - - from presets.registry import PresetRegistry - - # Create a test registry - preset_dir = repo_root / "configs/models" - registry = PresetRegistry(preset_dir) - - print("\n" + "-" * 80) - print("After initialization:") - print("-" * 80) - print(f" Available names: {len(registry._available_names)}") - print(f" Cached configs: {len(registry._cache)}") - assert len(registry._cache) == 0, "Cache should be empty initially" - print("✓ Cache is empty after initialization (lazy loading confirmed)") - - print("\n" + "-" * 80) - print("After loading one preset:") - print("-" * 80) - _ = registry.get("mask_rcnn_swin_s") - print(f" Available names: {len(registry._available_names)}") - print(f" Cached configs: {len(registry._cache)}") - assert len(registry._cache) == 1, "Cache should have one entry" - print("✓ Only loaded preset is cached") - - return True - - -if __name__ == "__main__": - print("\n🔍 Preset System Test Suite (Lightweight)\n") - - all_passed = True - - try: - if not test_preset_discovery(): - all_passed = False - if not test_preset_loading(): - all_passed = False - if not test_preset_registration(): - all_passed = False - if not test_lazy_loading(): - all_passed = False - except Exception as e: - print(f"\n❌ Test failed with error: {e}") - import traceback - - traceback.print_exc() - all_passed = False - - if all_passed: - print("\n" + "=" * 80) - print("🎉 ALL TESTS PASSED!") - print("=" * 80) - print("\nThe preset system is working correctly:") - print(" ✓ Preset discovery works") - print(" ✓ Preset loading works") - print(" ✓ Custom registration works") - print(" ✓ Lazy loading is confirmed") - print("\n") - sys.exit(0) - else: - print("\n" + "=" * 80) - print("❌ SOME TESTS FAILED") - print("=" * 80) - sys.exit(1) diff --git a/scripts/test_yaml_simple.py b/scripts/test_yaml_simple.py deleted file mode 100644 index 270e8af8..00000000 --- a/scripts/test_yaml_simple.py +++ /dev/null @@ -1,332 +0,0 @@ -#!/usr/bin/env python3 -"""Simplified test for YAML configuration loading. - -This test directly imports the YAML loader modules without importing -the full visdet package, avoiding import dependencies. -""" - -import sys -from pathlib import Path - -# Setup repo_root before imports -repo_root = Path(__file__).parent.parent - -# Add parent directory to path for direct module imports -sys.path.insert(0, str(repo_root / "visdet/visdet")) - -# Import YAML loader directly (avoid full visdet package import) -# ruff: noqa: E402 -from engine.config.yaml_loader import load_yaml_config - - -def test_component_loading(): - """Test loading individual component configs.""" - print("=" * 80) - print("Testing Individual Component YAML Loading") - print("=" * 80) - - configs_dir = repo_root / "configs/components" - - # Test backbone config - print("\n" + "-" * 80) - print("Test 1: Load Backbone Config") - print("-" * 80) - - backbone_path = configs_dir / "backbones/swin_tiny.yaml" - print(f"Loading: {backbone_path}") - - try: - cfg = load_yaml_config(backbone_path) - print("✓ Loaded successfully!") - print(f" - type: {cfg.type}") - print(f" - embed_dims: {cfg.embed_dims}") - print(f" - depths: {cfg.depths}") - print(f" - num_heads: {cfg.num_heads}") - assert cfg.type == "SwinTransformer" - assert cfg.embed_dims == 96 - assert cfg.depths == [2, 2, 6, 2] - except Exception as e: - print(f"✗ Failed: {e}") - import traceback - - traceback.print_exc() - return False - - # Test neck config - print("\n" + "-" * 80) - print("Test 2: Load Neck Config") - print("-" * 80) - - neck_path = configs_dir / "necks/fpn_256.yaml" - print(f"Loading: {neck_path}") - - try: - cfg = load_yaml_config(neck_path) - print("✓ Loaded successfully!") - print(f" - type: {cfg.type}") - print(f" - in_channels: {cfg.in_channels}") - print(f" - out_channels: {cfg.out_channels}") - print(f" - num_outs: {cfg.num_outs}") - assert cfg.type == "FPN" - assert cfg.out_channels == 256 - assert cfg.num_outs == 5 - except Exception as e: - print(f"✗ Failed: {e}") - import traceback - - traceback.print_exc() - return False - - # Test optimizer config - print("\n" + "-" * 80) - print("Test 3: Load Optimizer Config") - print("-" * 80) - - optimizer_path = configs_dir / "optimizers/adamw_default.yaml" - print(f"Loading: {optimizer_path}") - - try: - cfg = load_yaml_config(optimizer_path) - print("✓ Loaded successfully!") - print(f" - type: {cfg.type}") - print(f" - lr: {cfg.lr}") - print(f" - betas: {cfg.betas}") - print(f" - weight_decay: {cfg.weight_decay}") - assert cfg.type == "AdamW" - assert cfg.lr == 0.0001 - assert cfg.weight_decay == 0.05 - except Exception as e: - print(f"✗ Failed: {e}") - import traceback - - traceback.print_exc() - return False - - print("\n" + "=" * 80) - print("✓ All component loading tests passed!") - print("=" * 80) - return True - - -def test_base_inheritance(): - """Test _base_ inheritance mechanism.""" - print("\n" + "=" * 80) - print("Testing _base_ Inheritance") - print("=" * 80) - - experiment_path = repo_root / "configs/experiments/mask_rcnn_swin_tiny_coco.yaml" - print(f"\nLoading: {experiment_path}") - - try: - cfg = load_yaml_config(experiment_path) - print("✓ Loaded successfully!") - except Exception as e: - print(f"✗ Failed to load: {e}") - import traceback - - traceback.print_exc() - return False - - # Test that _base_ configs were merged - print("\n" + "-" * 80) - print("Checking _base_ Merged Configs") - print("-" * 80) - - # Check dataset config (from coco_instance.yaml) - if hasattr(cfg, "data_root") and cfg.data_root == "data/coco/": - print("✓ Dataset config inherited correctly") - print(f" - data_root: {cfg.data_root}") - print(f" - ann_file: {cfg.ann_file}") - else: - print("✗ Dataset config not inherited") - return False - - # Check optimizer config (from adamw_default.yaml) - if hasattr(cfg, "type") and cfg.type == "AdamW": - print("✓ Optimizer config inherited correctly") - print(f" - type: {cfg.type}") - print(f" - lr: {cfg.lr}") - else: - print("✗ Optimizer config not inherited") - return False - - # Check schedule config (from 1x.yaml) - if hasattr(cfg, "max_epochs"): - print("✓ Schedule config inherited correctly") - # Note: This should be 24 (overridden) not 12 (from base) - print(f" - max_epochs: {cfg.max_epochs}") - else: - print("✗ Schedule config not inherited") - return False - - # Test override mechanism - print("\n" + "-" * 80) - print("Checking Config Overrides") - print("-" * 80) - - if cfg.max_epochs == 24: - print("✓ Override works: max_epochs overridden to 24 (from 12 in base)") - else: - print(f"✗ Override failed: max_epochs = {cfg.max_epochs} (expected 24)") - return False - - if hasattr(cfg, "val_interval") and cfg.val_interval == 2: - print("✓ Override works: val_interval overridden to 2 (from 1 in base)") - else: - print(f"✗ Override failed: val_interval = {getattr(cfg, 'val_interval', 'N/A')}") - return False - - print("\n" + "=" * 80) - print("✓ All _base_ inheritance tests passed!") - print("=" * 80) - return True - - -def test_ref_resolution(): - """Test $ref reference resolution.""" - print("\n" + "=" * 80) - print("Testing $ref Resolution") - print("=" * 80) - - experiment_path = repo_root / "configs/experiments/mask_rcnn_swin_tiny_coco.yaml" - print(f"\nLoading: {experiment_path}") - - try: - cfg = load_yaml_config(experiment_path) - print("✓ Loaded successfully!") - except Exception as e: - print(f"✗ Failed to load: {e}") - return False - - # Check backbone $ref resolution - print("\n" + "-" * 80) - print("Checking Backbone $ref") - print("-" * 80) - - if hasattr(cfg, "model") and hasattr(cfg.model, "backbone"): - backbone = cfg.model.backbone - if backbone.type == "SwinTransformer" and backbone.embed_dims == 96: - print("✓ Backbone $ref resolved correctly") - print(f" - type: {backbone.type}") - print(f" - embed_dims: {backbone.embed_dims}") - print(f" - depths: {backbone.depths}") - else: - print("✗ Backbone $ref not resolved correctly") - return False - else: - print("✗ Model backbone not found") - return False - - # Check neck $ref resolution - print("\n" + "-" * 80) - print("Checking Neck $ref") - print("-" * 80) - - if hasattr(cfg.model, "neck"): - neck = cfg.model.neck - if neck.type == "FPN" and neck.out_channels == 256: - print("✓ Neck $ref resolved correctly") - print(f" - type: {neck.type}") - print(f" - out_channels: {neck.out_channels}") - print(f" - num_outs: {neck.num_outs}") - else: - print("✗ Neck $ref not resolved correctly") - return False - else: - print("✗ Model neck not found") - return False - - print("\n" + "=" * 80) - print("✓ All $ref resolution tests passed!") - print("=" * 80) - return True - - -def test_attribute_access(): - """Test ConfigDict attribute access.""" - print("\n" + "=" * 80) - print("Testing ConfigDict Attribute Access") - print("=" * 80) - - experiment_path = repo_root / "configs/experiments/mask_rcnn_swin_tiny_coco.yaml" - cfg = load_yaml_config(experiment_path) - - # Test attribute access - print("\n" + "-" * 80) - print("Test Attribute Access (dot notation)") - print("-" * 80) - - try: - assert cfg.model.type == "MaskRCNN" - assert cfg.model.backbone.type == "SwinTransformer" - assert cfg.model.neck.out_channels == 256 - assert cfg.max_epochs == 24 - print("✓ Attribute access works") - print(f" - cfg.model.type = {cfg.model.type}") - print(f" - cfg.model.backbone.type = {cfg.model.backbone.type}") - print(f" - cfg.max_epochs = {cfg.max_epochs}") - except (AttributeError, AssertionError) as e: - print(f"✗ Attribute access failed: {e}") - return False - - # Test dict-style access - print("\n" + "-" * 80) - print("Test Dict-Style Access (bracket notation)") - print("-" * 80) - - try: - assert cfg["model"]["type"] == "MaskRCNN" - assert cfg["model"]["backbone"]["type"] == "SwinTransformer" - assert cfg["model"]["neck"]["out_channels"] == 256 - assert cfg["max_epochs"] == 24 - print("✓ Dict-style access works") - print(f" - cfg['model']['type'] = {cfg['model']['type']}") - print(f" - cfg['model']['backbone']['type'] = {cfg['model']['backbone']['type']}") - print(f" - cfg['max_epochs'] = {cfg['max_epochs']}") - except (KeyError, AssertionError) as e: - print(f"✗ Dict-style access failed: {e}") - return False - - print("\n" + "=" * 80) - print("✓ All attribute access tests passed!") - print("=" * 80) - return True - - -if __name__ == "__main__": - print("\n🔍 YAML Configuration System Test Suite\n") - - all_passed = True - - # Run all tests - if not test_component_loading(): - all_passed = False - - if not test_base_inheritance(): - all_passed = False - - if not test_ref_resolution(): - all_passed = False - - if not test_attribute_access(): - all_passed = False - - # Final summary - if all_passed: - print("\n" + "=" * 80) - print("🎉 ALL TESTS PASSED!") - print("=" * 80) - print("\nThe YAML configuration system is working correctly:") - print(" ✓ Individual component configs load successfully") - print(" ✓ _base_ inheritance merges parent configs") - print(" ✓ Config overrides work as expected") - print(" ✓ $ref resolution loads referenced files") - print(" ✓ Attribute and dict-style access both work") - print("\n") - sys.exit(0) - else: - print("\n" + "=" * 80) - print("❌ SOME TESTS FAILED") - print("=" * 80) - print("\nPlease review the output above for details.\n") - sys.exit(1) diff --git a/tools/.ipynb_checkpoints/train-checkpoint.py b/tools/.ipynb_checkpoints/train-checkpoint.py deleted file mode 100644 index 27aa818e..00000000 --- a/tools/.ipynb_checkpoints/train-checkpoint.py +++ /dev/null @@ -1,247 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import copy -import os -import os.path as osp -import time -import warnings - -import mmcv -import torch -import torch.distributed as dist -from mmcv import Config, DictAction -from mmcv.runner import get_dist_info, init_dist -from mmcv.utils import get_git_hash - -from mmdet import __version__ -from mmdet.apis import init_random_seed, set_random_seed, train_detector -from mmdet.datasets import build_dataset -from mmdet.models import build_detector -from mmdet.utils import (collect_env, get_device, get_root_logger, - replace_cfg_vals, rfnext_init_model, - setup_multi_processes, update_data_root) - - -def parse_args(): - parser = argparse.ArgumentParser(description='Train a detector') - parser.add_argument('config', help='train config file path') - parser.add_argument('--work-dir', help='the dir to save logs and models') - parser.add_argument( - '--resume-from', help='the checkpoint file to resume from') - parser.add_argument( - '--auto-resume', - action='store_true', - help='resume from the latest checkpoint automatically') - parser.add_argument( - '--no-validate', - action='store_true', - help='whether not to evaluate the checkpoint during training') - group_gpus = parser.add_mutually_exclusive_group() - group_gpus.add_argument( - '--gpus', - type=int, - help='(Deprecated, please use --gpu-id) number of gpus to use ' - '(only applicable to non-distributed training)') - group_gpus.add_argument( - '--gpu-ids', - type=int, - nargs='+', - help='(Deprecated, please use --gpu-id) ids of gpus to use ' - '(only applicable to non-distributed training)') - group_gpus.add_argument( - '--gpu-id', - type=int, - default=0, - help='id of gpu to use ' - '(only applicable to non-distributed training)') - parser.add_argument('--seed', type=int, default=None, help='random seed') - parser.add_argument( - '--diff-seed', - action='store_true', - help='Whether or not set different seeds for different ranks') - parser.add_argument( - '--deterministic', - action='store_true', - help='whether to set deterministic options for CUDNN backend.') - parser.add_argument( - '--options', - nargs='+', - action=DictAction, - help='override some settings in the used config, the key-value pair ' - 'in xxx=yyy format will be merged into config file (deprecate), ' - 'change to --cfg-options instead.') - parser.add_argument( - '--cfg-options', - nargs='+', - action=DictAction, - help='override some settings in the used config, the key-value pair ' - 'in xxx=yyy format will be merged into config file. If the value to ' - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - 'Note that the quotation marks are necessary and that no white space ' - 'is allowed.') - parser.add_argument( - '--launcher', - choices=['none', 'pytorch', 'slurm', 'mpi'], - default='none', - help='job launcher') - parser.add_argument('--local_rank', type=int, default=0) - parser.add_argument( - '--auto-scale-lr', - action='store_true', - help='enable automatically scaling LR.') - args = parser.parse_args() - if 'LOCAL_RANK' not in os.environ: - os.environ['LOCAL_RANK'] = str(args.local_rank) - - if args.options and args.cfg_options: - raise ValueError( - '--options and --cfg-options cannot be both ' - 'specified, --options is deprecated in favor of --cfg-options') - if args.options: - warnings.warn('--options is deprecated in favor of --cfg-options') - args.cfg_options = args.options - - return args - - -def main(): - args = parse_args() - - cfg = Config.fromfile(args.config) - - # replace the ${key} with the value of cfg.key - cfg = replace_cfg_vals(cfg) - - # update data root according to MMDET_DATASETS - update_data_root(cfg) - - if args.cfg_options is not None: - cfg.merge_from_dict(args.cfg_options) - - if args.auto_scale_lr: - if 'auto_scale_lr' in cfg and \ - 'enable' in cfg.auto_scale_lr and \ - 'base_batch_size' in cfg.auto_scale_lr: - cfg.auto_scale_lr.enable = True - else: - warnings.warn('Can not find "auto_scale_lr" or ' - '"auto_scale_lr.enable" or ' - '"auto_scale_lr.base_batch_size" in your' - ' configuration file. Please update all the ' - 'configuration files to mmdet >= 2.24.1.') - - # set multi-process settings - setup_multi_processes(cfg) - - # set cudnn_benchmark - if cfg.get('cudnn_benchmark', False): - torch.backends.cudnn.benchmark = True - - # work_dir is determined in this priority: CLI > segment in file > filename - if args.work_dir is not None: - # update configs according to CLI args if args.work_dir is not None - cfg.work_dir = args.work_dir - elif cfg.get('work_dir', None) is None: - # use config filename as default work_dir if cfg.work_dir is None - cfg.work_dir = osp.join('./work_dirs', - osp.splitext(osp.basename(args.config))[0]) - - if args.resume_from is not None: - cfg.resume_from = args.resume_from - cfg.auto_resume = args.auto_resume - if args.gpus is not None: - cfg.gpu_ids = range(1) - warnings.warn('`--gpus` is deprecated because we only support ' - 'single GPU mode in non-distributed training. ' - 'Use `gpus=1` now.') - if args.gpu_ids is not None: - cfg.gpu_ids = args.gpu_ids[0:1] - warnings.warn('`--gpu-ids` is deprecated, please use `--gpu-id`. ' - 'Because we only support single GPU mode in ' - 'non-distributed training. Use the first GPU ' - 'in `gpu_ids` now.') - if args.gpus is None and args.gpu_ids is None: - cfg.gpu_ids = [args.gpu_id] - - # init distributed env first, since logger depends on the dist info. - if args.launcher == 'none': - distributed = False - else: - distributed = True - init_dist(args.launcher, **cfg.dist_params) - # re-set gpu_ids with distributed training mode - _, world_size = get_dist_info() - cfg.gpu_ids = range(world_size) - - # create work_dir - mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir)) - # dump config - cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config))) - # init the logger before other steps - timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime()) - log_file = osp.join(cfg.work_dir, f'{timestamp}.log') - logger = get_root_logger(log_file=log_file, log_level=cfg.log_level) - - # init the meta dict to record some important information such as - # environment info and seed, which will be logged - meta = dict() - # log env info - env_info_dict = collect_env() - env_info = '\n'.join([(f'{k}: {v}') for k, v in env_info_dict.items()]) - dash_line = '-' * 60 + '\n' - logger.info('Environment info:\n' + dash_line + env_info + '\n' + - dash_line) - meta['env_info'] = env_info - meta['config'] = cfg.pretty_text - # log some basic info - logger.info(f'Distributed training: {distributed}') - logger.info(f'Config:\n{cfg.pretty_text}') - - cfg.device = get_device() - # set random seeds - seed = init_random_seed(args.seed, device=cfg.device) - seed = seed + dist.get_rank() if args.diff_seed else seed - logger.info(f'Set random seed to {seed}, ' - f'deterministic: {args.deterministic}') - set_random_seed(seed, deterministic=args.deterministic) - cfg.seed = seed - meta['seed'] = seed - meta['exp_name'] = osp.basename(args.config) - - model = build_detector( - cfg.model, - train_cfg=cfg.get('train_cfg'), - test_cfg=cfg.get('test_cfg')) - model.init_weights() - - # init rfnext if 'RFSearchHook' is defined in cfg - rfnext_init_model(model, cfg=cfg) - - datasets = [build_dataset(cfg.data.train)] - if len(cfg.workflow) == 2: - assert 'val' in [mode for (mode, _) in cfg.workflow] - val_dataset = copy.deepcopy(cfg.data.val) - val_dataset.pipeline = cfg.data.train.get( - 'pipeline', cfg.data.train.dataset.get('pipeline')) - datasets.append(build_dataset(val_dataset)) - if cfg.checkpoint_config is not None: - # save mmdet version, config file content and class names in - # checkpoints as meta data - cfg.checkpoint_config.meta = dict( - mmdet_version=__version__ + get_git_hash()[:7], - CLASSES=datasets[0].CLASSES) - # add an attribute for visualization convenience - model.CLASSES = datasets[0].CLASSES - train_detector( - model, - datasets, - cfg, - distributed=distributed, - validate=(not args.no_validate), - timestamp=timestamp, - meta=meta) - - -if __name__ == '__main__': - main() diff --git a/tools/analysis_tools/analyze_logs.py b/tools/analysis_tools/analyze_logs.py deleted file mode 100755 index b9721ca9..00000000 --- a/tools/analysis_tools/analyze_logs.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import json -from collections import defaultdict - -import matplotlib.pyplot as plt -import numpy as np -import seaborn as sns - - -def cal_train_time(log_dicts, args): - for i, log_dict in enumerate(log_dicts): - print(f"{'-' * 5}Analyze train time of {args.json_logs[i]}{'-' * 5}") - all_times = [] - for epoch in log_dict.keys(): - if args.include_outliers: - all_times.append(log_dict[epoch]["time"]) - else: - all_times.append(log_dict[epoch]["time"][1:]) - if not all_times: - raise KeyError( - "Please reduce the log interval in the config so thatinterval is less than iterations of one epoch." - ) - all_times = np.array(all_times) - epoch_ave_time = all_times.mean(-1) - slowest_epoch = epoch_ave_time.argmax() - fastest_epoch = epoch_ave_time.argmin() - std_over_epoch = epoch_ave_time.std() - print(f"slowest epoch {slowest_epoch + 1}, average time is {epoch_ave_time[slowest_epoch]:.4f}") - print(f"fastest epoch {fastest_epoch + 1}, average time is {epoch_ave_time[fastest_epoch]:.4f}") - print(f"time std over epochs is {std_over_epoch:.4f}") - print(f"average iter time: {np.mean(all_times):.4f} s/iter") - print() - - -def plot_curve(log_dicts, args): - if args.backend is not None: - plt.switch_backend(args.backend) - sns.set_style(args.style) - # if legend is None, use {filename}_{key} as legend - legend = args.legend - if legend is None: - legend = [] - for json_log in args.json_logs: - for metric in args.keys: - legend.append(f"{json_log}_{metric}") - assert len(legend) == (len(args.json_logs) * len(args.keys)) - metrics = args.keys - - num_metrics = len(metrics) - for i, log_dict in enumerate(log_dicts): - epochs = list(log_dict.keys()) - for j, metric in enumerate(metrics): - print(f"plot curve of {args.json_logs[i]}, metric is {metric}") - if metric not in log_dict[epochs[int(args.eval_interval) - 1]]: - if "mAP" in metric: - raise KeyError( - f"{args.json_logs[i]} does not contain metric " - f'{metric}. Please check if "--no-validate" is ' - "specified when you trained the model." - ) - raise KeyError( - f"{args.json_logs[i]} does not contain metric {metric}. " - "Please reduce the log interval in the config so that " - "interval is less than iterations of one epoch." - ) - - if "mAP" in metric: - xs = [] - ys = [] - for epoch in epochs: - ys += log_dict[epoch][metric] - if "val" in log_dict[epoch]["mode"]: - xs.append(epoch) - plt.xlabel("epoch") - plt.plot(xs, ys, label=legend[i * num_metrics + j], marker="o") - else: - xs = [] - ys = [] - num_iters_per_epoch = log_dict[epochs[0]]["iter"][-2] - for epoch in epochs: - iters = log_dict[epoch]["iter"] - if log_dict[epoch]["mode"][-1] == "val": - iters = iters[:-1] - xs.append(np.array(iters) + (epoch - 1) * num_iters_per_epoch) - ys.append(np.array(log_dict[epoch][metric][: len(iters)])) - xs = np.concatenate(xs) - ys = np.concatenate(ys) - plt.xlabel("iter") - plt.plot(xs, ys, label=legend[i * num_metrics + j], linewidth=0.5) - plt.legend() - if args.title is not None: - plt.title(args.title) - if args.out is None: - plt.show() - else: - print(f"save curve to: {args.out}") - plt.savefig(args.out) - plt.cla() - - -def add_plot_parser(subparsers): - parser_plt = subparsers.add_parser("plot_curve", help="parser for plotting curves") - parser_plt.add_argument("json_logs", type=str, nargs="+", help="path of train log in json format") - parser_plt.add_argument( - "--keys", - type=str, - nargs="+", - default=["bbox_mAP"], - help="the metric that you want to plot", - ) - parser_plt.add_argument("--start-epoch", type=str, default="1", help="the epoch that you want to start") - parser_plt.add_argument("--eval-interval", type=str, default="1", help="the eval interval when training") - parser_plt.add_argument("--title", type=str, help="title of figure") - parser_plt.add_argument("--legend", type=str, nargs="+", default=None, help="legend of each plot") - parser_plt.add_argument("--backend", type=str, default=None, help="backend of plt") - parser_plt.add_argument("--style", type=str, default="dark", help="style of plt") - parser_plt.add_argument("--out", type=str, default=None) - - -def add_time_parser(subparsers): - parser_time = subparsers.add_parser( - "cal_train_time", - help="parser for computing the average time per training iteration", - ) - parser_time.add_argument("json_logs", type=str, nargs="+", help="path of train log in json format") - parser_time.add_argument( - "--include-outliers", - action="store_true", - help="include the first value of every epoch when computing the average time", - ) - - -def parse_args(): - parser = argparse.ArgumentParser(description="Analyze Json Log") - # currently only support plot curve and calculate average train time - subparsers = parser.add_subparsers(dest="task", help="task parser") - add_plot_parser(subparsers) - add_time_parser(subparsers) - args = parser.parse_args() - return args - - -def load_json_logs(json_logs): - # load and convert json_logs to log_dict, key is epoch, value is a sub dict - # keys of sub dict is different metrics, e.g. memory, bbox_mAP - # value of sub dict is a list of corresponding values of all iterations - log_dicts = [dict() for _ in json_logs] - for json_log, log_dict in zip(json_logs, log_dicts): - with open(json_log) as log_file: - for i, line in enumerate(log_file): - log = json.loads(line.strip()) - # skip the first training info line - if i == 0: - continue - # skip lines without `epoch` field - if "epoch" not in log: - continue - epoch = log.pop("epoch") - if epoch not in log_dict: - log_dict[epoch] = defaultdict(list) - for k, v in log.items(): - log_dict[epoch][k].append(v) - return log_dicts - - -def main(): - args = parse_args() - - json_logs = args.json_logs - for json_log in json_logs: - assert json_log.endswith(".json") - - log_dicts = load_json_logs(json_logs) - - eval(args.task)(log_dicts, args) - - -if __name__ == "__main__": - main() diff --git a/tools/analysis_tools/analyze_results.py b/tools/analysis_tools/analyze_results.py deleted file mode 100644 index bb5a4a15..00000000 --- a/tools/analysis_tools/analyze_results.py +++ /dev/null @@ -1,352 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import os.path as osp -from multiprocessing import Pool - -import mmcv -import numpy as np -from mmcv import Config, DictAction -from mmdet.core.evaluation import eval_map -from mmdet.core.visualization import imshow_gt_det_bboxes -from mmdet.datasets import build_dataset, get_loading_pipeline -from mmdet.datasets.api_wrappers import pq_compute_single_core -from mmdet.utils import replace_cfg_vals, update_data_root - - -def bbox_map_eval(det_result, annotation, nproc=4): - """Evaluate mAP of single image det result. - - Args: - det_result (list[list]): [[cls1_det, cls2_det, ...], ...]. - The outer list indicates images, and the inner list indicates - per-class detected bboxes. - annotation (dict): Ground truth annotations where keys of - annotations are: - - - bboxes: numpy array of shape (n, 4) - - labels: numpy array of shape (n, ) - - bboxes_ignore (optional): numpy array of shape (k, 4) - - labels_ignore (optional): numpy array of shape (k, ) - - nproc (int): Processes used for computing mAP. - Default: 4. - - Returns: - float: mAP - """ - - # use only bbox det result - if isinstance(det_result, tuple): - bbox_det_result = [det_result[0]] - else: - bbox_det_result = [det_result] - # mAP - iou_thrs = np.linspace(0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True) - - processes = [] - workers = Pool(processes=nproc) - for thr in iou_thrs: - p = workers.apply_async( - eval_map, - (bbox_det_result, [annotation]), - {"iou_thr": thr, "logger": "silent", "nproc": 1}, - ) - processes.append(p) - - workers.close() - workers.join() - - mean_aps = [] - for p in processes: - mean_aps.append(p.get()[0]) - - return sum(mean_aps) / len(mean_aps) - - -class ResultVisualizer: - """Display and save evaluation results. - - Args: - show (bool): Whether to show the image. Default: True. - wait_time (float): Value of waitKey param. Default: 0. - score_thr (float): Minimum score of bboxes to be shown. - Default: 0. - overlay_gt_pred (bool): Whether to plot gts and predictions on the - same image. If False, predictions and gts will be plotted on two - same image which will be concatenated in vertical direction. - The image above is drawn with gt, and the image below is drawn - with the prediction result. Default: False. - """ - - def __init__(self, show=False, wait_time=0, score_thr=0, overlay_gt_pred=False): - self.show = show - self.wait_time = wait_time - self.score_thr = score_thr - self.overlay_gt_pred = overlay_gt_pred - - def _save_image_gts_results(self, dataset, results, performances, out_dir=None): - """Display or save image with groung truths and predictions from a - model. - - Args: - dataset (Dataset): A PyTorch dataset. - results (list): Object detection or panoptic segmentation - results from test results pkl file. - performances (dict): A dict contains samples's indices - in dataset and model's performance on them. - out_dir (str, optional): The filename to write the image. - Defaults: None. - """ - mmcv.mkdir_or_exist(out_dir) - - for performance_info in performances: - index, performance = performance_info - data_info = dataset.prepare_train_img(index) - - # calc save file path - filename = data_info["filename"] - if data_info["img_prefix"] is not None: - filename = osp.join(data_info["img_prefix"], filename) - else: - filename = data_info["filename"] - fname, name = osp.splitext(osp.basename(filename)) - save_filename = fname + "_" + str(round(performance, 3)) + name - out_file = osp.join(out_dir, save_filename) - imshow_gt_det_bboxes( - data_info["img"], - data_info, - results[index], - dataset.CLASSES, - gt_bbox_color=dataset.PALETTE, - gt_text_color=(200, 200, 200), - gt_mask_color=dataset.PALETTE, - det_bbox_color=dataset.PALETTE, - det_text_color=(200, 200, 200), - det_mask_color=dataset.PALETTE, - show=self.show, - score_thr=self.score_thr, - wait_time=self.wait_time, - out_file=out_file, - overlay_gt_pred=self.overlay_gt_pred, - ) - - def evaluate_and_show(self, dataset, results, topk=20, show_dir="work_dir"): - """Evaluate and show results. - - Args: - dataset (Dataset): A PyTorch dataset. - results (list): Object detection or panoptic segmentation - results from test results pkl file. - topk (int): Number of the highest topk and - lowest topk after evaluation index sorting. Default: 20. - show_dir (str, optional): The filename to write the image. - Default: 'work_dir' - eval_fn (callable, optional): Eval function, Default: None. - """ - - assert topk > 0 - if (topk * 2) > len(dataset): - topk = len(dataset) // 2 - - if isinstance(results[0], dict): - good_samples, bad_samples = self.panoptic_evaluate(dataset, results, topk=topk) - elif isinstance(results[0], list): - good_samples, bad_samples = self.detection_evaluate(dataset, results, topk=topk) - elif isinstance(results[0], tuple): - results_ = [result[0] for result in results] - good_samples, bad_samples = self.detection_evaluate(dataset, results_, topk=topk) - else: - raise ( - "The format of result is not supported yet. " - "Current dict for panoptic segmentation and list " - "or tuple for object detection are supported." - ) - - good_dir = osp.abspath(osp.join(show_dir, "good")) - bad_dir = osp.abspath(osp.join(show_dir, "bad")) - self._save_image_gts_results(dataset, results, good_samples, good_dir) - self._save_image_gts_results(dataset, results, bad_samples, bad_dir) - - def detection_evaluate(self, dataset, results, topk=20, eval_fn=None): - """Evaluation for object detection. - - Args: - dataset (Dataset): A PyTorch dataset. - results (list): Object detection results from test - results pkl file. - topk (int): Number of the highest topk and - lowest topk after evaluation index sorting. Default: 20. - eval_fn (callable, optional): Eval function, Default: None. - - Returns: - tuple: A tuple contains good samples and bad samples. - good_mAPs (dict[int, float]): A dict contains good - samples's indices in dataset and model's - performance on them. - bad_mAPs (dict[int, float]): A dict contains bad - samples's indices in dataset and model's - performance on them. - """ - if eval_fn is None: - eval_fn = bbox_map_eval - else: - assert callable(eval_fn) - - prog_bar = mmcv.ProgressBar(len(results)) - _mAPs = {} - for i, (result,) in enumerate(zip(results)): - # self.dataset[i] should not call directly - # because there is a risk of mismatch - data_info = dataset.prepare_train_img(i) - mAP = eval_fn(result, data_info["ann_info"]) - _mAPs[i] = mAP - prog_bar.update() - # descending select topk image - _mAPs = list(sorted(_mAPs.items(), key=lambda kv: kv[1])) - good_mAPs = _mAPs[-topk:] - bad_mAPs = _mAPs[:topk] - - return good_mAPs, bad_mAPs - - def panoptic_evaluate(self, dataset, results, topk=20): - """Evaluation for panoptic segmentation. - - Args: - dataset (Dataset): A PyTorch dataset. - results (list): Panoptic segmentation results from test - results pkl file. - topk (int): Number of the highest topk and - lowest topk after evaluation index sorting. Default: 20. - - Returns: - tuple: A tuple contains good samples and bad samples. - good_pqs (dict[int, float]): A dict contains good - samples's indices in dataset and model's - performance on them. - bad_pqs (dict[int, float]): A dict contains bad - samples's indices in dataset and model's - performance on them. - """ - # image to annotations - gt_json = dataset.coco.img_ann_map - - result_files, tmp_dir = dataset.format_results(results) - pred_json = mmcv.load(result_files["panoptic"])["annotations"] - pred_folder = osp.join(tmp_dir.name, "panoptic") - gt_folder = dataset.seg_prefix - - pqs = {} - prog_bar = mmcv.ProgressBar(len(results)) - for i in range(len(results)): - data_info = dataset.prepare_train_img(i) - image_id = data_info["img_info"]["id"] - gt_ann = { - "image_id": image_id, - "segments_info": gt_json[image_id], - "file_name": data_info["img_info"]["segm_file"], - } - pred_ann = pred_json[i] - pq_stat = pq_compute_single_core( - i, - [(gt_ann, pred_ann)], - gt_folder, - pred_folder, - dataset.categories, - dataset.file_client, - print_log=False, - ) - pq_results, classwise_results = pq_stat.pq_average(dataset.categories, isthing=None) - pqs[i] = pq_results["pq"] - prog_bar.update() - - if tmp_dir is not None: - tmp_dir.cleanup() - - # descending select topk image - pqs = list(sorted(pqs.items(), key=lambda kv: kv[1])) - good_pqs = pqs[-topk:] - bad_pqs = pqs[:topk] - - return good_pqs, bad_pqs - - -def parse_args(): - parser = argparse.ArgumentParser(description="MMDet eval image prediction result for each") - parser.add_argument("config", help="test config file path") - parser.add_argument("prediction_path", help="prediction path where test pkl result") - parser.add_argument("show_dir", help="directory where painted images will be saved") - parser.add_argument("--show", action="store_true", help="show results") - parser.add_argument( - "--wait-time", - type=float, - default=0, - help="the interval of show (s), 0 is block", - ) - parser.add_argument( - "--topk", - default=20, - type=int, - help="saved Number of the highest topk and lowest topk after index sorting", - ) - parser.add_argument("--show-score-thr", type=float, default=0, help="score threshold (default: 0.)") - parser.add_argument( - "--overlay-gt-pred", - action="store_true", - help="whether to plot gts and predictions on the same image." - "If False, predictions and gts will be plotted on two same" - "image which will be concatenated in vertical direction." - "The image above is drawn with gt, and the image below is" - "drawn with the prediction result.", - ) - parser.add_argument( - "--cfg-options", - nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file. If the value to " - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - "Note that the quotation marks are necessary and that no white space " - "is allowed.", - ) - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - - mmcv.check_file_exist(args.prediction_path) - - cfg = Config.fromfile(args.config) - - # replace the ${key} with the value of cfg.key - cfg = replace_cfg_vals(cfg) - - # update data root according to MMDET_DATASETS - update_data_root(cfg) - - if args.cfg_options is not None: - cfg.merge_from_dict(args.cfg_options) - cfg.data.test.test_mode = True - - cfg.data.test.pop("samples_per_gpu", 0) - if cfg.data.train.type in ( - "MultiImageMixDataset", - "ClassBalancedDataset", - "RepeatDataset", - "ConcatDataset", - ): - cfg.data.test.pipeline = get_loading_pipeline(cfg.data.train.dataset.pipeline) - else: - cfg.data.test.pipeline = get_loading_pipeline(cfg.data.train.pipeline) - - dataset = build_dataset(cfg.data.test) - outputs = mmcv.load(args.prediction_path) - - result_visualizer = ResultVisualizer(args.show, args.wait_time, args.show_score_thr, args.overlay_gt_pred) - result_visualizer.evaluate_and_show(dataset, outputs, topk=args.topk, show_dir=args.show_dir) - - -if __name__ == "__main__": - main() diff --git a/tools/analysis_tools/benchmark.py b/tools/analysis_tools/benchmark.py deleted file mode 100644 index 8e3f0499..00000000 --- a/tools/analysis_tools/benchmark.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import copy -import os -import time - -import torch -from mmcv import Config, DictAction -from mmcv.cnn import fuse_conv_bn -from mmcv.parallel import MMDistributedDataParallel -from mmcv.runner import init_dist, load_checkpoint, wrap_fp16_model -from mmdet.datasets import build_dataloader, build_dataset, replace_ImageToTensor -from mmdet.models import build_detector -from mmdet.utils import replace_cfg_vals, update_data_root - - -def parse_args(): - parser = argparse.ArgumentParser(description="MMDet benchmark a model") - parser.add_argument("config", help="test config file path") - parser.add_argument("checkpoint", help="checkpoint file") - parser.add_argument( - "--repeat-num", - type=int, - default=1, - help="number of repeat times of measurement for averaging the results", - ) - parser.add_argument("--max-iter", type=int, default=2000, help="num of max iter") - parser.add_argument("--log-interval", type=int, default=50, help="interval of logging") - parser.add_argument( - "--fuse-conv-bn", - action="store_true", - help="Whether to fuse conv and bn, this will slightly increasethe inference speed", - ) - parser.add_argument( - "--cfg-options", - nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file. If the value to " - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - "Note that the quotation marks are necessary and that no white space " - "is allowed.", - ) - parser.add_argument( - "--launcher", - choices=["none", "pytorch", "slurm", "mpi"], - default="none", - help="job launcher", - ) - parser.add_argument("--local_rank", type=int, default=0) - args = parser.parse_args() - if "LOCAL_RANK" not in os.environ: - os.environ["LOCAL_RANK"] = str(args.local_rank) - return args - - -def measure_inference_speed(cfg, checkpoint, max_iter, log_interval, is_fuse_conv_bn): - # set cudnn_benchmark - if cfg.get("cudnn_benchmark", False): - torch.backends.cudnn.benchmark = True - cfg.model.pretrained = None - cfg.data.test.test_mode = True - - # build the dataloader - samples_per_gpu = cfg.data.test.pop("samples_per_gpu", 1) - if samples_per_gpu > 1: - # Replace 'ImageToTensor' to 'DefaultFormatBundle' - cfg.data.test.pipeline = replace_ImageToTensor(cfg.data.test.pipeline) - dataset = build_dataset(cfg.data.test) - data_loader = build_dataloader( - dataset, - samples_per_gpu=1, - # Because multiple processes will occupy additional CPU resources, - # FPS statistics will be more unstable when workers_per_gpu is not 0. - # It is reasonable to set workers_per_gpu to 0. - workers_per_gpu=0, - dist=True, - shuffle=False, - ) - - # build the model and load checkpoint - cfg.model.train_cfg = None - model = build_detector(cfg.model, test_cfg=cfg.get("test_cfg")) - fp16_cfg = cfg.get("fp16", None) - if fp16_cfg is not None: - wrap_fp16_model(model) - load_checkpoint(model, checkpoint, map_location="cpu") - if is_fuse_conv_bn: - model = fuse_conv_bn(model) - - model = MMDistributedDataParallel(model.cuda(), device_ids=[torch.cuda.current_device()], broadcast_buffers=False) - model.eval() - - # the first several iterations may be very slow so skip them - num_warmup = 5 - pure_inf_time = 0 - fps = 0 - - # benchmark with 2000 image and take the average - for i, data in enumerate(data_loader): - torch.cuda.synchronize() - start_time = time.perf_counter() - - with torch.no_grad(): - model(return_loss=False, rescale=True, **data) - - torch.cuda.synchronize() - elapsed = time.perf_counter() - start_time - - if i >= num_warmup: - pure_inf_time += elapsed - if (i + 1) % log_interval == 0: - fps = (i + 1 - num_warmup) / pure_inf_time - print( - f"Done image [{i + 1:<3}/ {max_iter}], " - f"fps: {fps:.1f} img / s, " - f"times per image: {1000 / fps:.1f} ms / img", - flush=True, - ) - - if (i + 1) == max_iter: - fps = (i + 1 - num_warmup) / pure_inf_time - print( - f"Overall fps: {fps:.1f} img / s, times per image: {1000 / fps:.1f} ms / img", - flush=True, - ) - break - return fps - - -def repeat_measure_inference_speed(cfg, checkpoint, max_iter, log_interval, is_fuse_conv_bn, repeat_num=1): - assert repeat_num >= 1 - - fps_list = [] - - for _ in range(repeat_num): - # - cp_cfg = copy.deepcopy(cfg) - - fps_list.append(measure_inference_speed(cp_cfg, checkpoint, max_iter, log_interval, is_fuse_conv_bn)) - - if repeat_num > 1: - fps_list_ = [round(fps, 1) for fps in fps_list] - times_pre_image_list_ = [round(1000 / fps, 1) for fps in fps_list] - mean_fps_ = sum(fps_list_) / len(fps_list_) - mean_times_pre_image_ = sum(times_pre_image_list_) / len(times_pre_image_list_) - print( - f"Overall fps: {fps_list_}[{mean_fps_:.1f}] img / s, " - f"times per image: " - f"{times_pre_image_list_}[{mean_times_pre_image_:.1f}] ms / img", - flush=True, - ) - return fps_list - - return fps_list[0] - - -def main(): - args = parse_args() - - cfg = Config.fromfile(args.config) - - # replace the ${key} with the value of cfg.key - cfg = replace_cfg_vals(cfg) - - # update data root according to MMDET_DATASETS - update_data_root(cfg) - - if args.cfg_options is not None: - cfg.merge_from_dict(args.cfg_options) - - if args.launcher == "none": - raise NotImplementedError("Only supports distributed mode") - else: - init_dist(args.launcher, **cfg.dist_params) - - repeat_measure_inference_speed( - cfg, - args.checkpoint, - args.max_iter, - args.log_interval, - args.fuse_conv_bn, - args.repeat_num, - ) - - -if __name__ == "__main__": - main() diff --git a/tools/analysis_tools/coco_error_analysis.py b/tools/analysis_tools/coco_error_analysis.py deleted file mode 100644 index bedb6a0b..00000000 --- a/tools/analysis_tools/coco_error_analysis.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import copy -import os -from argparse import ArgumentParser -from multiprocessing import Pool - -import matplotlib.pyplot as plt -import numpy as np -from pycocotools.coco import COCO -from pycocotools.cocoeval import COCOeval - - -def makeplot(rs, ps, outDir, class_name, iou_type): - cs = np.vstack( - [ - np.ones((2, 3)), - np.array([0.31, 0.51, 0.74]), - np.array([0.75, 0.31, 0.30]), - np.array([0.36, 0.90, 0.38]), - np.array([0.50, 0.39, 0.64]), - np.array([1, 0.6, 0]), - ] - ) - areaNames = ["allarea", "small", "medium", "large"] - types = ["C75", "C50", "Loc", "Sim", "Oth", "BG", "FN"] - for i in range(len(areaNames)): - area_ps = ps[..., i, 0] - figure_title = iou_type + "-" + class_name + "-" + areaNames[i] - aps = [ps_.mean() for ps_ in area_ps] - ps_curve = [ps_.mean(axis=1) if ps_.ndim > 1 else ps_ for ps_ in area_ps] - ps_curve.insert(0, np.zeros(ps_curve[0].shape)) - fig = plt.figure() - ax = plt.subplot(111) - for k in range(len(types)): - ax.plot(rs, ps_curve[k + 1], color=[0, 0, 0], linewidth=0.5) - ax.fill_between( - rs, - ps_curve[k], - ps_curve[k + 1], - color=cs[k], - label=str(f"[{aps[k]:.3f}]" + types[k]), - ) - plt.xlabel("recall") - plt.ylabel("precision") - plt.xlim(0, 1.0) - plt.ylim(0, 1.0) - plt.title(figure_title) - plt.legend() - # plt.show() - fig.savefig(outDir + f"/{figure_title}.png") - plt.close(fig) - - -def autolabel(ax, rects): - """Attach a text label above each bar in *rects*, displaying its height.""" - for rect in rects: - height = rect.get_height() - if height > 0 and height <= 1: # for percent values - text_label = f"{height * 100:2.0f}" - else: - text_label = f"{height:2.0f}" - ax.annotate( - text_label, - xy=(rect.get_x() + rect.get_width() / 2, height), - xytext=(0, 3), # 3 points vertical offset - textcoords="offset points", - ha="center", - va="bottom", - fontsize="x-small", - ) - - -def makebarplot(rs, ps, outDir, class_name, iou_type): - areaNames = ["allarea", "small", "medium", "large"] - types = ["C75", "C50", "Loc", "Sim", "Oth", "BG", "FN"] - fig, ax = plt.subplots() - x = np.arange(len(areaNames)) # the areaNames locations - width = 0.60 # the width of the bars - rects_list = [] - figure_title = iou_type + "-" + class_name + "-" + "ap bar plot" - for i in range(len(types) - 1): - type_ps = ps[i, ..., 0] - aps = [ps_.mean() for ps_ in type_ps.T] - rects_list.append( - ax.bar( - x - width / 2 + (i + 1) * width / len(types), - aps, - width / len(types), - label=types[i], - ) - ) - - # Add some text for labels, title and custom x-axis tick labels, etc. - ax.set_ylabel("Mean Average Precision (mAP)") - ax.set_title(figure_title) - ax.set_xticks(x) - ax.set_xticklabels(areaNames) - ax.legend() - - # Add score texts over bars - for rects in rects_list: - autolabel(ax, rects) - - # Save plot - fig.savefig(outDir + f"/{figure_title}.png") - plt.close(fig) - - -def get_gt_area_group_numbers(cocoEval): - areaRng = cocoEval.params.areaRng - areaRngStr = [str(aRng) for aRng in areaRng] - areaRngLbl = cocoEval.params.areaRngLbl - areaRngStr2areaRngLbl = dict(zip(areaRngStr, areaRngLbl)) - areaRngLbl2Number = dict.fromkeys(areaRngLbl, 0) - for evalImg in cocoEval.evalImgs: - if evalImg: - for gtIgnore in evalImg["gtIgnore"]: - if not gtIgnore: - aRngLbl = areaRngStr2areaRngLbl[str(evalImg["aRng"])] - areaRngLbl2Number[aRngLbl] += 1 - return areaRngLbl2Number - - -def make_gt_area_group_numbers_plot(cocoEval, outDir, verbose=True): - areaRngLbl2Number = get_gt_area_group_numbers(cocoEval) - areaRngLbl = areaRngLbl2Number.keys() - if verbose: - print("number of annotations per area group:", areaRngLbl2Number) - - # Init figure - fig, ax = plt.subplots() - x = np.arange(len(areaRngLbl)) # the areaNames locations - width = 0.60 # the width of the bars - figure_title = "number of annotations per area group" - - rects = ax.bar(x, areaRngLbl2Number.values(), width) - - # Add some text for labels, title and custom x-axis tick labels, etc. - ax.set_ylabel("Number of annotations") - ax.set_title(figure_title) - ax.set_xticks(x) - ax.set_xticklabels(areaRngLbl) - - # Add score texts over bars - autolabel(ax, rects) - - # Save plot - fig.tight_layout() - fig.savefig(outDir + f"/{figure_title}.png") - plt.close(fig) - - -def make_gt_area_histogram_plot(cocoEval, outDir): - n_bins = 100 - areas = [ann["area"] for ann in cocoEval.cocoGt.anns.values()] - - # init figure - figure_title = "gt annotation areas histogram plot" - fig, ax = plt.subplots() - - # Set the number of bins - ax.hist(np.sqrt(areas), bins=n_bins) - - # Add some text for labels, title and custom x-axis tick labels, etc. - ax.set_xlabel("Squareroot Area") - ax.set_ylabel("Number of annotations") - ax.set_title(figure_title) - - # Save plot - fig.tight_layout() - fig.savefig(outDir + f"/{figure_title}.png") - plt.close(fig) - - -def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type, areas=None): - nm = cocoGt.loadCats(catId)[0] - print(f"--------------analyzing {k + 1}-{nm['name']}---------------") - ps_ = {} - dt = copy.deepcopy(cocoDt) - nm = cocoGt.loadCats(catId)[0] - imgIds = cocoGt.getImgIds() - dt_anns = dt.dataset["annotations"] - select_dt_anns = [] - for ann in dt_anns: - if ann["category_id"] == catId: - select_dt_anns.append(ann) - dt.dataset["annotations"] = select_dt_anns - dt.createIndex() - # compute precision but ignore superclass confusion - gt = copy.deepcopy(cocoGt) - child_catIds = gt.getCatIds(supNms=[nm["supercategory"]]) - for idx, ann in enumerate(gt.dataset["annotations"]): - if ann["category_id"] in child_catIds and ann["category_id"] != catId: - gt.dataset["annotations"][idx]["ignore"] = 1 - gt.dataset["annotations"][idx]["iscrowd"] = 1 - gt.dataset["annotations"][idx]["category_id"] = catId - cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type) - cocoEval.params.imgIds = imgIds - cocoEval.params.maxDets = [100] - cocoEval.params.iouThrs = [0.1] - cocoEval.params.useCats = 1 - if areas: - cocoEval.params.areaRng = [ - [0**2, areas[2]], - [0**2, areas[0]], - [areas[0], areas[1]], - [areas[1], areas[2]], - ] - cocoEval.evaluate() - cocoEval.accumulate() - ps_supercategory = cocoEval.eval["precision"][0, :, k, :, :] - ps_["ps_supercategory"] = ps_supercategory - # compute precision but ignore any class confusion - gt = copy.deepcopy(cocoGt) - for idx, ann in enumerate(gt.dataset["annotations"]): - if ann["category_id"] != catId: - gt.dataset["annotations"][idx]["ignore"] = 1 - gt.dataset["annotations"][idx]["iscrowd"] = 1 - gt.dataset["annotations"][idx]["category_id"] = catId - cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type) - cocoEval.params.imgIds = imgIds - cocoEval.params.maxDets = [100] - cocoEval.params.iouThrs = [0.1] - cocoEval.params.useCats = 1 - if areas: - cocoEval.params.areaRng = [ - [0**2, areas[2]], - [0**2, areas[0]], - [areas[0], areas[1]], - [areas[1], areas[2]], - ] - cocoEval.evaluate() - cocoEval.accumulate() - ps_allcategory = cocoEval.eval["precision"][0, :, k, :, :] - ps_["ps_allcategory"] = ps_allcategory - return k, ps_ - - -def analyze_results(res_file, ann_file, res_types, out_dir, extraplots=None, areas=None): - for res_type in res_types: - assert res_type in ["bbox", "segm"] - if areas: - assert len(areas) == 3, ( - "3 integers should be specified as areas, \ - representing 3 area regions" - ) - - directory = os.path.dirname(out_dir + "/") - if not os.path.exists(directory): - print(f"-------------create {out_dir}-----------------") - os.makedirs(directory) - - cocoGt = COCO(ann_file) - cocoDt = cocoGt.loadRes(res_file) - imgIds = cocoGt.getImgIds() - for res_type in res_types: - res_out_dir = out_dir + "/" + res_type + "/" - res_directory = os.path.dirname(res_out_dir) - if not os.path.exists(res_directory): - print(f"-------------create {res_out_dir}-----------------") - os.makedirs(res_directory) - iou_type = res_type - cocoEval = COCOeval(copy.deepcopy(cocoGt), copy.deepcopy(cocoDt), iou_type) - cocoEval.params.imgIds = imgIds - cocoEval.params.iouThrs = [0.75, 0.5, 0.1] - cocoEval.params.maxDets = [100] - if areas: - cocoEval.params.areaRng = [ - [0**2, areas[2]], - [0**2, areas[0]], - [areas[0], areas[1]], - [areas[1], areas[2]], - ] - cocoEval.evaluate() - cocoEval.accumulate() - ps = cocoEval.eval["precision"] - ps = np.vstack([ps, np.zeros((4, *ps.shape[1:]))]) - catIds = cocoGt.getCatIds() - recThrs = cocoEval.params.recThrs - with Pool(processes=48) as pool: - args = [(k, cocoDt, cocoGt, catId, iou_type, areas) for k, catId in enumerate(catIds)] - analyze_results = pool.starmap(analyze_individual_category, args) - for k, catId in enumerate(catIds): - nm = cocoGt.loadCats(catId)[0] - print(f"--------------saving {k + 1}-{nm['name']}---------------") - analyze_result = analyze_results[k] - assert k == analyze_result[0] - ps_supercategory = analyze_result[1]["ps_supercategory"] - ps_allcategory = analyze_result[1]["ps_allcategory"] - # compute precision but ignore superclass confusion - ps[3, :, k, :, :] = ps_supercategory - # compute precision but ignore any class confusion - ps[4, :, k, :, :] = ps_allcategory - # fill in background and false negative errors and plot - ps[ps == -1] = 0 - ps[5, :, k, :, :] = ps[4, :, k, :, :] > 0 - ps[6, :, k, :, :] = 1.0 - makeplot(recThrs, ps[:, :, k], res_out_dir, nm["name"], iou_type) - if extraplots: - makebarplot(recThrs, ps[:, :, k], res_out_dir, nm["name"], iou_type) - makeplot(recThrs, ps, res_out_dir, "allclass", iou_type) - if extraplots: - makebarplot(recThrs, ps, res_out_dir, "allclass", iou_type) - make_gt_area_group_numbers_plot(cocoEval=cocoEval, outDir=res_out_dir, verbose=True) - make_gt_area_histogram_plot(cocoEval=cocoEval, outDir=res_out_dir) - - -def main(): - parser = ArgumentParser(description="COCO Error Analysis Tool") - parser.add_argument("result", help="result file (json format) path") - parser.add_argument("out_dir", help="dir to save analyze result images") - parser.add_argument( - "--ann", - default="data/coco/annotations/instances_val2017.json", - help="annotation file path", - ) - parser.add_argument("--types", type=str, nargs="+", default=["bbox"], help="result types") - parser.add_argument("--extraplots", action="store_true", help="export extra bar/stat plots") - parser.add_argument( - "--areas", - type=int, - nargs="+", - default=[1024, 9216, 10000000000], - help="area regions", - ) - args = parser.parse_args() - analyze_results( - args.result, - args.ann, - args.types, - out_dir=args.out_dir, - extraplots=args.extraplots, - areas=args.areas, - ) - - -if __name__ == "__main__": - main() diff --git a/tools/analysis_tools/coco_occluded_separated_recall.py b/tools/analysis_tools/coco_occluded_separated_recall.py deleted file mode 100644 index ea09a434..00000000 --- a/tools/analysis_tools/coco_occluded_separated_recall.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -from argparse import ArgumentParser - -import mmcv -from mmcv.utils import print_log -from mmdet.datasets import OccludedSeparatedCocoDataset - - -def main(): - parser = ArgumentParser( - description="Compute recall of COCO occluded and separated masks " - "presented in paper https://arxiv.org/abs/2210.10046." - ) - parser.add_argument("result", help="result file (pkl format) path") - parser.add_argument("--out", help="file path to save evaluation results") - parser.add_argument( - "--score-thr", - type=float, - default=0.3, - help="Score threshold for the recall calculation. Defaults to 0.3", - ) - parser.add_argument( - "--iou-thr", - type=float, - default=0.75, - help="IoU threshold for the recall calculation. Defaults to 0.75.", - ) - parser.add_argument( - "--ann", - default="data/coco/annotations/instances_val2017.json", - help="coco annotation file path", - ) - args = parser.parse_args() - - results = mmcv.load(args.result) - assert isinstance(results[0], tuple), "The results must be predicted by instance segmentation model." - dataset = OccludedSeparatedCocoDataset(ann_file=args.ann, pipeline=[], test_mode=True) - metric_res = dataset.evaluate(results) - if args.out is not None: - mmcv.dump(metric_res, args.out) - print_log(f"Evaluation results have been saved to {args.out}.") - - -if __name__ == "__main__": - main() diff --git a/tools/analysis_tools/confusion_matrix.py b/tools/analysis_tools/confusion_matrix.py deleted file mode 100644 index 45f95893..00000000 --- a/tools/analysis_tools/confusion_matrix.py +++ /dev/null @@ -1,267 +0,0 @@ -import argparse -import os - -import matplotlib.pyplot as plt -import mmcv -import numpy as np -from matplotlib.ticker import MultipleLocator -from mmcv import Config, DictAction -from mmcv.ops import nms -from mmdet.core.evaluation.bbox_overlaps import bbox_overlaps -from mmdet.datasets import build_dataset -from mmdet.utils import replace_cfg_vals, update_data_root - - -def parse_args(): - parser = argparse.ArgumentParser(description="Generate confusion matrix from detection results") - parser.add_argument("config", help="test config file path") - parser.add_argument("prediction_path", help="prediction path where test .pkl result") - parser.add_argument("save_dir", help="directory where confusion matrix will be saved") - parser.add_argument("--show", action="store_true", help="show confusion matrix") - parser.add_argument("--color-theme", default="plasma", help="theme of the matrix color map") - parser.add_argument( - "--score-thr", - type=float, - default=0.3, - help="score threshold to filter detection bboxes", - ) - parser.add_argument( - "--tp-iou-thr", - type=float, - default=0.5, - help="IoU threshold to be considered as matched", - ) - parser.add_argument( - "--nms-iou-thr", - type=float, - default=None, - help="nms IoU threshold, only applied when users want to change thenms IoU threshold.", - ) - parser.add_argument( - "--cfg-options", - nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file. If the value to " - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - "Note that the quotation marks are necessary and that no white space " - "is allowed.", - ) - args = parser.parse_args() - return args - - -def calculate_confusion_matrix(dataset, results, score_thr=0, nms_iou_thr=None, tp_iou_thr=0.5): - """Calculate the confusion matrix. - - Args: - dataset (Dataset): Test or val dataset. - results (list[ndarray]): A list of detection results in each image. - score_thr (float|optional): Score threshold to filter bboxes. - Default: 0. - nms_iou_thr (float|optional): nms IoU threshold, the detection results - have done nms in the detector, only applied when users want to - change the nms IoU threshold. Default: None. - tp_iou_thr (float|optional): IoU threshold to be considered as matched. - Default: 0.5. - """ - num_classes = len(dataset.CLASSES) - confusion_matrix = np.zeros(shape=[num_classes + 1, num_classes + 1]) - assert len(dataset) == len(results) - prog_bar = mmcv.ProgressBar(len(results)) - for idx, per_img_res in enumerate(results): - if isinstance(per_img_res, tuple): - res_bboxes, _ = per_img_res - else: - res_bboxes = per_img_res - ann = dataset.get_ann_info(idx) - gt_bboxes = ann["bboxes"] - labels = ann["labels"] - analyze_per_img_dets( - confusion_matrix, - gt_bboxes, - labels, - res_bboxes, - score_thr, - tp_iou_thr, - nms_iou_thr, - ) - prog_bar.update() - return confusion_matrix - - -def analyze_per_img_dets( - confusion_matrix, - gt_bboxes, - gt_labels, - result, - score_thr=0, - tp_iou_thr=0.5, - nms_iou_thr=None, -): - """Analyze detection results on each image. - - Args: - confusion_matrix (ndarray): The confusion matrix, - has shape (num_classes + 1, num_classes + 1). - gt_bboxes (ndarray): Ground truth bboxes, has shape (num_gt, 4). - gt_labels (ndarray): Ground truth labels, has shape (num_gt). - result (ndarray): Detection results, has shape - (num_classes, num_bboxes, 5). - score_thr (float): Score threshold to filter bboxes. - Default: 0. - tp_iou_thr (float): IoU threshold to be considered as matched. - Default: 0.5. - nms_iou_thr (float|optional): nms IoU threshold, the detection results - have done nms in the detector, only applied when users want to - change the nms IoU threshold. Default: None. - """ - true_positives = np.zeros_like(gt_labels) - for det_label, det_bboxes in enumerate(result): - if nms_iou_thr: - det_bboxes, _ = nms( - det_bboxes[:, :4], - det_bboxes[:, -1], - nms_iou_thr, - score_threshold=score_thr, - ) - ious = bbox_overlaps(det_bboxes[:, :4], gt_bboxes) - for i, det_bbox in enumerate(det_bboxes): - score = det_bbox[4] - det_match = 0 - if score >= score_thr: - for j, gt_label in enumerate(gt_labels): - if ious[i, j] >= tp_iou_thr: - det_match += 1 - if gt_label == det_label: - true_positives[j] += 1 # TP - confusion_matrix[gt_label, det_label] += 1 - if det_match == 0: # BG FP - confusion_matrix[-1, det_label] += 1 - for num_tp, gt_label in zip(true_positives, gt_labels): - if num_tp == 0: # FN - confusion_matrix[gt_label, -1] += 1 - - -def plot_confusion_matrix( - confusion_matrix, - labels, - save_dir=None, - show=True, - title="Normalized Confusion Matrix", - color_theme="plasma", -): - """Draw confusion matrix with matplotlib. - - Args: - confusion_matrix (ndarray): The confusion matrix. - labels (list[str]): List of class names. - save_dir (str|optional): If set, save the confusion matrix plot to the - given path. Default: None. - show (bool): Whether to show the plot. Default: True. - title (str): Title of the plot. Default: `Normalized Confusion Matrix`. - color_theme (str): Theme of the matrix color map. Default: `plasma`. - """ - # normalize the confusion matrix - per_label_sums = confusion_matrix.sum(axis=1)[:, np.newaxis] - confusion_matrix = confusion_matrix.astype(np.float32) / per_label_sums * 100 - - num_classes = len(labels) - fig, ax = plt.subplots(figsize=(0.5 * num_classes, 0.5 * num_classes * 0.8), dpi=180) - cmap = plt.get_cmap(color_theme) - im = ax.imshow(confusion_matrix, cmap=cmap) - plt.colorbar(mappable=im, ax=ax) - - title_font = {"weight": "bold", "size": 12} - ax.set_title(title, fontdict=title_font) - label_font = {"size": 10} - plt.ylabel("Ground Truth Label", fontdict=label_font) - plt.xlabel("Prediction Label", fontdict=label_font) - - # draw locator - xmajor_locator = MultipleLocator(1) - xminor_locator = MultipleLocator(0.5) - ax.xaxis.set_major_locator(xmajor_locator) - ax.xaxis.set_minor_locator(xminor_locator) - ymajor_locator = MultipleLocator(1) - yminor_locator = MultipleLocator(0.5) - ax.yaxis.set_major_locator(ymajor_locator) - ax.yaxis.set_minor_locator(yminor_locator) - - # draw grid - ax.grid(True, which="minor", linestyle="-") - - # draw label - ax.set_xticks(np.arange(num_classes)) - ax.set_yticks(np.arange(num_classes)) - ax.set_xticklabels(labels) - ax.set_yticklabels(labels) - - ax.tick_params(axis="x", bottom=False, top=True, labelbottom=False, labeltop=True) - plt.setp(ax.get_xticklabels(), rotation=45, ha="left", rotation_mode="anchor") - - # draw confution matrix value - for i in range(num_classes): - for j in range(num_classes): - ax.text( - j, - i, - f"{int(confusion_matrix[i, j]) if not np.isnan(confusion_matrix[i, j]) else -1}%", - ha="center", - va="center", - color="w", - size=7, - ) - - ax.set_ylim(len(confusion_matrix) - 0.5, -0.5) # matplotlib>3.1.1 - - fig.tight_layout() - if save_dir is not None: - plt.savefig(os.path.join(save_dir, "confusion_matrix.png"), format="png") - if show: - plt.show() - - -def main(): - args = parse_args() - - cfg = Config.fromfile(args.config) - - # replace the ${key} with the value of cfg.key - cfg = replace_cfg_vals(cfg) - - # update data root according to MMDET_DATASETS - update_data_root(cfg) - - if args.cfg_options is not None: - cfg.merge_from_dict(args.cfg_options) - - results = mmcv.load(args.prediction_path) - assert isinstance(results, list) - if isinstance(results[0], list): - pass - elif isinstance(results[0], tuple): - results = [result[0] for result in results] - else: - raise TypeError("invalid type of prediction results") - - if isinstance(cfg.data.test, dict): - cfg.data.test.test_mode = True - elif isinstance(cfg.data.test, list): - for ds_cfg in cfg.data.test: - ds_cfg.test_mode = True - dataset = build_dataset(cfg.data.test) - - confusion_matrix = calculate_confusion_matrix(dataset, results, args.score_thr, args.nms_iou_thr, args.tp_iou_thr) - plot_confusion_matrix( - confusion_matrix, - dataset.CLASSES + ("background",), - save_dir=args.save_dir, - show=args.show, - color_theme=args.color_theme, - ) - - -if __name__ == "__main__": - main() diff --git a/tools/analysis_tools/eval_metric.py b/tools/analysis_tools/eval_metric.py deleted file mode 100644 index 304b94ac..00000000 --- a/tools/analysis_tools/eval_metric.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse - -import mmcv -from mmcv import Config, DictAction -from mmdet.datasets import build_dataset -from mmdet.utils import replace_cfg_vals, update_data_root - - -def parse_args(): - parser = argparse.ArgumentParser(description="Evaluate metric of the results saved in pkl format") - parser.add_argument("config", help="Config of the model") - parser.add_argument("pkl_results", help="Results in pickle format") - parser.add_argument( - "--format-only", - action="store_true", - help="Format the output results without perform evaluation. It is" - "useful when you want to format the result to a specific format and " - "submit it to the test server", - ) - parser.add_argument( - "--eval", - type=str, - nargs="+", - help='Evaluation metrics, which depends on the dataset, e.g., "bbox",' - ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC', - ) - parser.add_argument( - "--cfg-options", - nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file. If the value to " - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - "Note that the quotation marks are necessary and that no white space " - "is allowed.", - ) - parser.add_argument( - "--eval-options", - nargs="+", - action=DictAction, - help="custom options for evaluation, the key-value pair in xxx=yyy " - "format will be kwargs for dataset.evaluate() function", - ) - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - - cfg = Config.fromfile(args.config) - - # replace the ${key} with the value of cfg.key - cfg = replace_cfg_vals(cfg) - - # update data root according to MMDET_DATASETS - update_data_root(cfg) - - assert args.eval or args.format_only, ( - 'Please specify at least one operation (eval/format the results) with the argument "--eval", "--format-only"' - ) - if args.eval and args.format_only: - raise ValueError("--eval and --format_only cannot be both specified") - - if args.cfg_options is not None: - cfg.merge_from_dict(args.cfg_options) - cfg.data.test.test_mode = True - - dataset = build_dataset(cfg.data.test) - outputs = mmcv.load(args.pkl_results) - - kwargs = {} if args.eval_options is None else args.eval_options - if args.format_only: - dataset.format_results(outputs, **kwargs) - if args.eval: - eval_kwargs = cfg.get("evaluation", {}).copy() - # hard-code way to remove EvalHook args - for key in ["interval", "tmpdir", "start", "gpu_collect", "save_best", "rule"]: - eval_kwargs.pop(key, None) - eval_kwargs.update(dict(metric=args.eval, **kwargs)) - print(dataset.evaluate(outputs, **eval_kwargs)) - - -if __name__ == "__main__": - main() diff --git a/tools/analysis_tools/get_flops.py b/tools/analysis_tools/get_flops.py deleted file mode 100644 index 8822613e..00000000 --- a/tools/analysis_tools/get_flops.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse - -import numpy as np -import torch -from mmcv import Config, DictAction -from mmdet.models import build_detector - -try: - from mmcv.cnn import get_model_complexity_info -except ImportError: - raise ImportError("Please upgrade mmcv to >0.6.2") - - -def parse_args(): - parser = argparse.ArgumentParser(description="Train a detector") - parser.add_argument("config", help="train config file path") - parser.add_argument("--shape", type=int, nargs="+", default=[1280, 800], help="input image size") - parser.add_argument( - "--cfg-options", - nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file. If the value to " - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - "Note that the quotation marks are necessary and that no white space " - "is allowed.", - ) - parser.add_argument( - "--size-divisor", - type=int, - default=32, - help="Pad the input image, the minimum size that is divisible by size_divisor, -1 means do not pad the image.", - ) - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - - if len(args.shape) == 1: - h = w = args.shape[0] - elif len(args.shape) == 2: - h, w = args.shape - else: - raise ValueError("invalid input shape") - ori_shape = (3, h, w) - divisor = args.size_divisor - if divisor > 0: - h = int(np.ceil(h / divisor)) * divisor - w = int(np.ceil(w / divisor)) * divisor - - input_shape = (3, h, w) - - cfg = Config.fromfile(args.config) - if args.cfg_options is not None: - cfg.merge_from_dict(args.cfg_options) - - model = build_detector(cfg.model, train_cfg=cfg.get("train_cfg"), test_cfg=cfg.get("test_cfg")) - if torch.cuda.is_available(): - model.cuda() - model.eval() - - if hasattr(model, "forward_dummy"): - model.forward = model.forward_dummy - else: - raise NotImplementedError(f"FLOPs counter is currently not currently supported with {model.__class__.__name__}") - - flops, params = get_model_complexity_info(model, input_shape) - split_line = "=" * 30 - - if divisor > 0 and input_shape != ori_shape: - print(f"{split_line}\nUse size divisor set input shape from {ori_shape} to {input_shape}\n") - print(f"{split_line}\nInput shape: {input_shape}\nFlops: {flops}\nParams: {params}\n{split_line}") - print( - "!!!Please be cautious if you use the results in papers. " - "You may need to check if all ops are supported and verify that the " - "flops computation is correct." - ) - - -if __name__ == "__main__": - main() diff --git a/tools/analysis_tools/optimize_anchors.py b/tools/analysis_tools/optimize_anchors.py deleted file mode 100644 index 802e0177..00000000 --- a/tools/analysis_tools/optimize_anchors.py +++ /dev/null @@ -1,357 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -"""Optimize anchor settings on a specific dataset. - -This script provides two method to optimize YOLO anchors including k-means -anchor cluster and differential evolution. You can use ``--algorithm k-means`` -and ``--algorithm differential_evolution`` to switch two method. - -Example: - Use k-means anchor cluster:: - - python tools/analysis_tools/optimize_anchors.py ${CONFIG} \ - --algorithm k-means --input-shape ${INPUT_SHAPE [WIDTH HEIGHT]} \ - --output-dir ${OUTPUT_DIR} - Use differential evolution to optimize anchors:: - - python tools/analysis_tools/optimize_anchors.py ${CONFIG} \ - --algorithm differential_evolution \ - --input-shape ${INPUT_SHAPE [WIDTH HEIGHT]} \ - --output-dir ${OUTPUT_DIR} -""" - -import argparse -import os.path as osp - -import mmcv -import numpy as np -import torch -from mmcv import Config -from mmdet.core import bbox_cxcywh_to_xyxy, bbox_overlaps, bbox_xyxy_to_cxcywh -from mmdet.datasets import build_dataset -from mmdet.utils import get_root_logger, replace_cfg_vals, update_data_root -from scipy.optimize import differential_evolution - - -def parse_args(): - parser = argparse.ArgumentParser(description="Optimize anchor parameters.") - parser.add_argument("config", help="Train config file path.") - parser.add_argument("--device", default="cuda:0", help="Device used for calculating.") - parser.add_argument( - "--input-shape", - type=int, - nargs="+", - default=[608, 608], - help="input image size", - ) - parser.add_argument( - "--algorithm", - default="differential_evolution", - help="Algorithm used for anchor optimizing.Support k-means and differential_evolution for YOLO.", - ) - parser.add_argument("--iters", default=1000, type=int, help="Maximum iterations for optimizer.") - parser.add_argument( - "--output-dir", - default=None, - type=str, - help="Path to save anchor optimize result.", - ) - - args = parser.parse_args() - return args - - -class BaseAnchorOptimizer: - """Base class for anchor optimizer. - - Args: - dataset (obj:`Dataset`): Dataset object. - input_shape (list[int]): Input image shape of the model. - Format in [width, height]. - logger (obj:`logging.Logger`): The logger for logging. - device (str, optional): Device used for calculating. - Default: 'cuda:0' - out_dir (str, optional): Path to save anchor optimize result. - Default: None - """ - - def __init__(self, dataset, input_shape, logger, device="cuda:0", out_dir=None): - self.dataset = dataset - self.input_shape = input_shape - self.logger = logger - self.device = device - self.out_dir = out_dir - bbox_whs, img_shapes = self.get_whs_and_shapes() - ratios = img_shapes.max(1, keepdims=True) / np.array([input_shape]) - - # resize to input shape - self.bbox_whs = bbox_whs / ratios - - def get_whs_and_shapes(self): - """Get widths and heights of bboxes and shapes of images. - - Returns: - tuple[np.ndarray]: Array of bbox shapes and array of image - shapes with shape (num_bboxes, 2) in [width, height] format. - """ - self.logger.info("Collecting bboxes from annotation...") - bbox_whs = [] - img_shapes = [] - prog_bar = mmcv.ProgressBar(len(self.dataset)) - for idx in range(len(self.dataset)): - ann = self.dataset.get_ann_info(idx) - data_info = self.dataset.data_infos[idx] - img_shape = np.array([data_info["width"], data_info["height"]]) - gt_bboxes = ann["bboxes"] - for bbox in gt_bboxes: - wh = bbox[2:4] - bbox[0:2] - img_shapes.append(img_shape) - bbox_whs.append(wh) - prog_bar.update() - print("\n") - bbox_whs = np.array(bbox_whs) - img_shapes = np.array(img_shapes) - self.logger.info(f"Collected {bbox_whs.shape[0]} bboxes.") - return bbox_whs, img_shapes - - def get_zero_center_bbox_tensor(self): - """Get a tensor of bboxes centered at (0, 0). - - Returns: - Tensor: Tensor of bboxes with shape (num_bboxes, 4) - in [xmin, ymin, xmax, ymax] format. - """ - whs = torch.from_numpy(self.bbox_whs).to(self.device, dtype=torch.float32) - bboxes = bbox_cxcywh_to_xyxy(torch.cat([torch.zeros_like(whs), whs], dim=1)) - return bboxes - - def optimize(self): - raise NotImplementedError - - def save_result(self, anchors, path=None): - anchor_results = [] - for w, h in anchors: - anchor_results.append([round(w), round(h)]) - self.logger.info(f"Anchor optimize result:{anchor_results}") - if path: - json_path = osp.join(path, "anchor_optimize_result.json") - mmcv.dump(anchor_results, json_path) - self.logger.info(f"Result saved in {json_path}") - - -class YOLOKMeansAnchorOptimizer(BaseAnchorOptimizer): - r"""YOLO anchor optimizer using k-means. Code refer to `AlexeyAB/darknet. - `_. - - Args: - num_anchors (int) : Number of anchors. - iters (int): Maximum iterations for k-means. - """ - - def __init__(self, num_anchors, iters, **kwargs): - super(YOLOKMeansAnchorOptimizer, self).__init__(**kwargs) - self.num_anchors = num_anchors - self.iters = iters - - def optimize(self): - anchors = self.kmeans_anchors() - self.save_result(anchors, self.out_dir) - - def kmeans_anchors(self): - self.logger.info(f"Start cluster {self.num_anchors} YOLO anchors with K-means...") - bboxes = self.get_zero_center_bbox_tensor() - cluster_center_idx = torch.randint(0, bboxes.shape[0], (self.num_anchors,)).to(self.device) - - assignments = torch.zeros((bboxes.shape[0],)).to(self.device) - cluster_centers = bboxes[cluster_center_idx] - if self.num_anchors == 1: - cluster_centers = self.kmeans_maximization(bboxes, assignments, cluster_centers) - anchors = bbox_xyxy_to_cxcywh(cluster_centers)[:, 2:].cpu().numpy() - anchors = sorted(anchors, key=lambda x: x[0] * x[1]) - return anchors - - prog_bar = mmcv.ProgressBar(self.iters) - for i in range(self.iters): - converged, assignments = self.kmeans_expectation(bboxes, assignments, cluster_centers) - if converged: - self.logger.info(f"K-means process has converged at iter {i}.") - break - cluster_centers = self.kmeans_maximization(bboxes, assignments, cluster_centers) - prog_bar.update() - print("\n") - avg_iou = bbox_overlaps(bboxes, cluster_centers).max(1)[0].mean().item() - - anchors = bbox_xyxy_to_cxcywh(cluster_centers)[:, 2:].cpu().numpy() - anchors = sorted(anchors, key=lambda x: x[0] * x[1]) - self.logger.info(f"Anchor cluster finish. Average IOU: {avg_iou}") - - return anchors - - def kmeans_maximization(self, bboxes, assignments, centers): - """Maximization part of EM algorithm(Expectation-Maximization)""" - new_centers = torch.zeros_like(centers) - for i in range(centers.shape[0]): - mask = assignments == i - if mask.sum(): - new_centers[i, :] = bboxes[mask].mean(0) - return new_centers - - def kmeans_expectation(self, bboxes, assignments, centers): - """Expectation part of EM algorithm(Expectation-Maximization)""" - ious = bbox_overlaps(bboxes, centers) - closest = ious.argmax(1) - converged = (closest == assignments).all() - return converged, closest - - -class YOLODEAnchorOptimizer(BaseAnchorOptimizer): - """YOLO anchor optimizer using differential evolution algorithm. - - Args: - num_anchors (int) : Number of anchors. - iters (int): Maximum iterations for k-means. - strategy (str): The differential evolution strategy to use. - Should be one of: - - - 'best1bin' - - 'best1exp' - - 'rand1exp' - - 'randtobest1exp' - - 'currenttobest1exp' - - 'best2exp' - - 'rand2exp' - - 'randtobest1bin' - - 'currenttobest1bin' - - 'best2bin' - - 'rand2bin' - - 'rand1bin' - - Default: 'best1bin'. - population_size (int): Total population size of evolution algorithm. - Default: 15. - convergence_thr (float): Tolerance for convergence, the - optimizing stops when ``np.std(pop) <= abs(convergence_thr) - + convergence_thr * np.abs(np.mean(population_energies))``, - respectively. Default: 0.0001. - mutation (tuple[float]): Range of dithering randomly changes the - mutation constant. Default: (0.5, 1). - recombination (float): Recombination constant of crossover probability. - Default: 0.7. - """ - - def __init__( - self, - num_anchors, - iters, - strategy="best1bin", - population_size=15, - convergence_thr=0.0001, - mutation=(0.5, 1), - recombination=0.7, - **kwargs, - ): - super(YOLODEAnchorOptimizer, self).__init__(**kwargs) - - self.num_anchors = num_anchors - self.iters = iters - self.strategy = strategy - self.population_size = population_size - self.convergence_thr = convergence_thr - self.mutation = mutation - self.recombination = recombination - - def optimize(self): - anchors = self.differential_evolution() - self.save_result(anchors, self.out_dir) - - def differential_evolution(self): - bboxes = self.get_zero_center_bbox_tensor() - - bounds = [] - for i in range(self.num_anchors): - bounds.extend([(0, self.input_shape[0]), (0, self.input_shape[1])]) - - result = differential_evolution( - func=self.avg_iou_cost, - bounds=bounds, - args=(bboxes,), - strategy=self.strategy, - maxiter=self.iters, - popsize=self.population_size, - tol=self.convergence_thr, - mutation=self.mutation, - recombination=self.recombination, - updating="immediate", - disp=True, - ) - self.logger.info(f"Anchor evolution finish. Average IOU: {1 - result.fun}") - anchors = [(w, h) for w, h in zip(result.x[::2], result.x[1::2])] - anchors = sorted(anchors, key=lambda x: x[0] * x[1]) - return anchors - - @staticmethod - def avg_iou_cost(anchor_params, bboxes): - assert len(anchor_params) % 2 == 0 - anchor_whs = torch.tensor([[w, h] for w, h in zip(anchor_params[::2], anchor_params[1::2])]).to( - bboxes.device, dtype=bboxes.dtype - ) - anchor_boxes = bbox_cxcywh_to_xyxy(torch.cat([torch.zeros_like(anchor_whs), anchor_whs], dim=1)) - ious = bbox_overlaps(bboxes, anchor_boxes) - max_ious, _ = ious.max(1) - cost = 1 - max_ious.mean().item() - return cost - - -def main(): - logger = get_root_logger() - args = parse_args() - cfg = args.config - cfg = Config.fromfile(cfg) - - # replace the ${key} with the value of cfg.key - cfg = replace_cfg_vals(cfg) - - # update data root according to MMDET_DATASETS - update_data_root(cfg) - - input_shape = args.input_shape - assert len(input_shape) == 2 - - anchor_type = cfg.model.bbox_head.anchor_generator.type - assert anchor_type == "YOLOAnchorGenerator", f"Only support optimize YOLOAnchor, but get {anchor_type}." - - base_sizes = cfg.model.bbox_head.anchor_generator.base_sizes - num_anchors = sum([len(sizes) for sizes in base_sizes]) - - train_data_cfg = cfg.data.train - while "dataset" in train_data_cfg: - train_data_cfg = train_data_cfg["dataset"] - dataset = build_dataset(train_data_cfg) - - if args.algorithm == "k-means": - optimizer = YOLOKMeansAnchorOptimizer( - dataset=dataset, - input_shape=input_shape, - device=args.device, - num_anchors=num_anchors, - iters=args.iters, - logger=logger, - out_dir=args.output_dir, - ) - elif args.algorithm == "differential_evolution": - optimizer = YOLODEAnchorOptimizer( - dataset=dataset, - input_shape=input_shape, - device=args.device, - num_anchors=num_anchors, - iters=args.iters, - logger=logger, - out_dir=args.output_dir, - ) - else: - raise NotImplementedError(f"Only support k-means and differential_evolution, but get {args.algorithm}") - - optimizer.optimize() - - -if __name__ == "__main__": - main() diff --git a/tools/analysis_tools/robustness_eval.py b/tools/analysis_tools/robustness_eval.py deleted file mode 100644 index 064a63e4..00000000 --- a/tools/analysis_tools/robustness_eval.py +++ /dev/null @@ -1,270 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import os.path as osp -from argparse import ArgumentParser - -import mmcv -import numpy as np - - -def print_coco_results(results): - def _print(result, ap=1, iouThr=None, areaRng="all", maxDets=100): - titleStr = "Average Precision" if ap == 1 else "Average Recall" - typeStr = "(AP)" if ap == 1 else "(AR)" - iouStr = "0.50:0.95" if iouThr is None else f"{iouThr:0.2f}" - iStr = f" {titleStr:<18} {typeStr} @[ IoU={iouStr:<9} | " - iStr += f"area={areaRng:>6s} | maxDets={maxDets:>3d} ] = {result:0.3f}" - print(iStr) - - stats = np.zeros((12,)) - stats[0] = _print(results[0], 1) - stats[1] = _print(results[1], 1, iouThr=0.5) - stats[2] = _print(results[2], 1, iouThr=0.75) - stats[3] = _print(results[3], 1, areaRng="small") - stats[4] = _print(results[4], 1, areaRng="medium") - stats[5] = _print(results[5], 1, areaRng="large") - stats[6] = _print(results[6], 0, maxDets=1) - stats[7] = _print(results[7], 0, maxDets=10) - stats[8] = _print(results[8], 0) - stats[9] = _print(results[9], 0, areaRng="small") - stats[10] = _print(results[10], 0, areaRng="medium") - stats[11] = _print(results[11], 0, areaRng="large") - - -def get_coco_style_results(filename, task="bbox", metric=None, prints="mPC", aggregate="benchmark"): - assert aggregate in ["benchmark", "all"] - - if prints == "all": - prints = ["P", "mPC", "rPC"] - elif isinstance(prints, str): - prints = [prints] - for p in prints: - assert p in ["P", "mPC", "rPC"] - - if metric is None: - metrics = [ - "AP", - "AP50", - "AP75", - "APs", - "APm", - "APl", - "AR1", - "AR10", - "AR100", - "ARs", - "ARm", - "ARl", - ] - elif isinstance(metric, list): - metrics = metric - else: - metrics = [metric] - - for metric_name in metrics: - assert metric_name in [ - "AP", - "AP50", - "AP75", - "APs", - "APm", - "APl", - "AR1", - "AR10", - "AR100", - "ARs", - "ARm", - "ARl", - ] - - eval_output = mmcv.load(filename) - - num_distortions = len(list(eval_output.keys())) - results = np.zeros((num_distortions, 6, len(metrics)), dtype="float32") - - for corr_i, distortion in enumerate(eval_output): - for severity in eval_output[distortion]: - for metric_j, metric_name in enumerate(metrics): - mAP = eval_output[distortion][severity][task][metric_name] - results[corr_i, severity, metric_j] = mAP - - P = results[0, 0, :] - if aggregate == "benchmark": - mPC = np.mean(results[:15, 1:, :], axis=(0, 1)) - else: - mPC = np.mean(results[:, 1:, :], axis=(0, 1)) - rPC = mPC / P - - print(f"\nmodel: {osp.basename(filename)}") - if metric is None: - if "P" in prints: - print(f"Performance on Clean Data [P] ({task})") - print_coco_results(P) - if "mPC" in prints: - print(f"Mean Performance under Corruption [mPC] ({task})") - print_coco_results(mPC) - if "rPC" in prints: - print(f"Relative Performance under Corruption [rPC] ({task})") - print_coco_results(rPC) - else: - if "P" in prints: - print(f"Performance on Clean Data [P] ({task})") - for metric_i, metric_name in enumerate(metrics): - print(f"{metric_name:5} = {P[metric_i]:0.3f}") - if "mPC" in prints: - print(f"Mean Performance under Corruption [mPC] ({task})") - for metric_i, metric_name in enumerate(metrics): - print(f"{metric_name:5} = {mPC[metric_i]:0.3f}") - if "rPC" in prints: - print(f"Relative Performance under Corruption [rPC] ({task})") - for metric_i, metric_name in enumerate(metrics): - print(f"{metric_name:5} => {rPC[metric_i] * 100:0.1f} %") - - return results - - -def get_voc_style_results(filename, prints="mPC", aggregate="benchmark"): - assert aggregate in ["benchmark", "all"] - - if prints == "all": - prints = ["P", "mPC", "rPC"] - elif isinstance(prints, str): - prints = [prints] - for p in prints: - assert p in ["P", "mPC", "rPC"] - - eval_output = mmcv.load(filename) - - num_distortions = len(list(eval_output.keys())) - results = np.zeros((num_distortions, 6, 20), dtype="float32") - - for i, distortion in enumerate(eval_output): - for severity in eval_output[distortion]: - mAP = [eval_output[distortion][severity][j]["ap"] for j in range(len(eval_output[distortion][severity]))] - results[i, severity, :] = mAP - - P = results[0, 0, :] - if aggregate == "benchmark": - mPC = np.mean(results[:15, 1:, :], axis=(0, 1)) - else: - mPC = np.mean(results[:, 1:, :], axis=(0, 1)) - rPC = mPC / P - - print(f"\nmodel: {osp.basename(filename)}") - if "P" in prints: - print(f"Performance on Clean Data [P] in AP50 = {np.mean(P):0.3f}") - if "mPC" in prints: - print(f"Mean Performance under Corruption [mPC] in AP50 = {np.mean(mPC):0.3f}") - if "rPC" in prints: - print(f"Relative Performance under Corruption [rPC] in % = {np.mean(rPC) * 100:0.1f}") - - return np.mean(results, axis=2, keepdims=True) - - -def get_results( - filename, - dataset="coco", - task="bbox", - metric=None, - prints="mPC", - aggregate="benchmark", -): - assert dataset in ["coco", "voc", "cityscapes"] - - if dataset in ["coco", "cityscapes"]: - results = get_coco_style_results(filename, task=task, metric=metric, prints=prints, aggregate=aggregate) - elif dataset == "voc": - if task != "bbox": - print("Only bbox analysis is supported for Pascal VOC") - print("Will report bbox results\n") - if metric not in [None, ["AP"], ["AP50"]]: - print("Only the AP50 metric is supported for Pascal VOC") - print("Will report AP50 metric\n") - results = get_voc_style_results(filename, prints=prints, aggregate=aggregate) - - return results - - -def get_distortions_from_file(filename): - eval_output = mmcv.load(filename) - - return get_distortions_from_results(eval_output) - - -def get_distortions_from_results(eval_output): - distortions = [] - for i, distortion in enumerate(eval_output): - distortions.append(distortion.replace("_", " ")) - return distortions - - -def main(): - parser = ArgumentParser(description="Corruption Result Analysis") - parser.add_argument("filename", help="result file path") - parser.add_argument( - "--dataset", - type=str, - choices=["coco", "voc", "cityscapes"], - default="coco", - help="dataset type", - ) - parser.add_argument( - "--task", - type=str, - nargs="+", - choices=["bbox", "segm"], - default=["bbox"], - help="task to report", - ) - parser.add_argument( - "--metric", - nargs="+", - choices=[ - None, - "AP", - "AP50", - "AP75", - "APs", - "APm", - "APl", - "AR1", - "AR10", - "AR100", - "ARs", - "ARm", - "ARl", - ], - default=None, - help="metric to report", - ) - parser.add_argument( - "--prints", - type=str, - nargs="+", - choices=["P", "mPC", "rPC"], - default="mPC", - help="corruption benchmark metric to print", - ) - parser.add_argument( - "--aggregate", - type=str, - choices=["all", "benchmark"], - default="benchmark", - help="aggregate all results or only those \ - for benchmark corruptions", - ) - - args = parser.parse_args() - - for task in args.task: - get_results( - args.filename, - dataset=args.dataset, - task=task, - metric=args.metric, - prints=args.prints, - aggregate=args.aggregate, - ) - - -if __name__ == "__main__": - main() diff --git a/tools/dataset_converters/cityscapes.py b/tools/dataset_converters/cityscapes.py deleted file mode 100644 index 40bd944e..00000000 --- a/tools/dataset_converters/cityscapes.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import glob -import os.path as osp - -import cityscapesscripts.helpers.labels as CSLabels -import mmcv -import numpy as np -import pycocotools.mask as maskUtils - - -def collect_files(img_dir, gt_dir): - suffix = "leftImg8bit.png" - files = [] - for img_file in glob.glob(osp.join(img_dir, "**/*.png")): - assert img_file.endswith(suffix), img_file - inst_file = gt_dir + img_file[len(img_dir) : -len(suffix)] + "gtFine_instanceIds.png" - # Note that labelIds are not converted to trainId for seg map - segm_file = gt_dir + img_file[len(img_dir) : -len(suffix)] + "gtFine_labelIds.png" - files.append((img_file, inst_file, segm_file)) - assert len(files), f"No images found in {img_dir}" - print(f"Loaded {len(files)} images from {img_dir}") - - return files - - -def collect_annotations(files, nproc=1): - print("Loading annotation images") - if nproc > 1: - images = mmcv.track_parallel_progress(load_img_info, files, nproc=nproc) - else: - images = mmcv.track_progress(load_img_info, files) - - return images - - -def load_img_info(files): - img_file, inst_file, segm_file = files - inst_img = mmcv.imread(inst_file, "unchanged") - # ids < 24 are stuff labels (filtering them first is about 5% faster) - unique_inst_ids = np.unique(inst_img[inst_img >= 24]) - anno_info = [] - for inst_id in unique_inst_ids: - # For non-crowd annotations, inst_id // 1000 is the label_id - # Crowd annotations have <1000 instance ids - label_id = inst_id // 1000 if inst_id >= 1000 else inst_id - label = CSLabels.id2label[label_id] - if not label.hasInstances or label.ignoreInEval: - continue - - category_id = label.id - iscrowd = int(inst_id < 1000) - mask = np.asarray(inst_img == inst_id, dtype=np.uint8, order="F") - mask_rle = maskUtils.encode(mask[:, :, None])[0] - - area = maskUtils.area(mask_rle) - # convert to COCO style XYWH format - bbox = maskUtils.toBbox(mask_rle) - - # for json encoding - mask_rle["counts"] = mask_rle["counts"].decode() - - anno = dict( - iscrowd=iscrowd, - category_id=category_id, - bbox=bbox.tolist(), - area=area.tolist(), - segmentation=mask_rle, - ) - anno_info.append(anno) - video_name = osp.basename(osp.dirname(img_file)) - img_info = dict( - # remove img_prefix for filename - file_name=osp.join(video_name, osp.basename(img_file)), - height=inst_img.shape[0], - width=inst_img.shape[1], - anno_info=anno_info, - segm_file=osp.join(video_name, osp.basename(segm_file)), - ) - - return img_info - - -def cvt_annotations(image_infos, out_json_name): - out_json = dict() - img_id = 0 - ann_id = 0 - out_json["images"] = [] - out_json["categories"] = [] - out_json["annotations"] = [] - for image_info in image_infos: - image_info["id"] = img_id - anno_infos = image_info.pop("anno_info") - out_json["images"].append(image_info) - for anno_info in anno_infos: - anno_info["image_id"] = img_id - anno_info["id"] = ann_id - out_json["annotations"].append(anno_info) - ann_id += 1 - img_id += 1 - for label in CSLabels.labels: - if label.hasInstances and not label.ignoreInEval: - cat = dict(id=label.id, name=label.name) - out_json["categories"].append(cat) - - if len(out_json["annotations"]) == 0: - out_json.pop("annotations") - - mmcv.dump(out_json, out_json_name) - return out_json - - -def parse_args(): - parser = argparse.ArgumentParser(description="Convert Cityscapes annotations to COCO format") - parser.add_argument("cityscapes_path", help="cityscapes data path") - parser.add_argument("--img-dir", default="leftImg8bit", type=str) - parser.add_argument("--gt-dir", default="gtFine", type=str) - parser.add_argument("-o", "--out-dir", help="output path") - parser.add_argument("--nproc", default=1, type=int, help="number of process") - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - cityscapes_path = args.cityscapes_path - out_dir = args.out_dir if args.out_dir else cityscapes_path - mmcv.mkdir_or_exist(out_dir) - - img_dir = osp.join(cityscapes_path, args.img_dir) - gt_dir = osp.join(cityscapes_path, args.gt_dir) - - set_name = dict( - train="instancesonly_filtered_gtFine_train.json", - val="instancesonly_filtered_gtFine_val.json", - test="instancesonly_filtered_gtFine_test.json", - ) - - for split, json_name in set_name.items(): - print(f"Converting {split} into {json_name}") - with mmcv.Timer(print_tmpl="It took {}s to convert Cityscapes annotation"): - files = collect_files(osp.join(img_dir, split), osp.join(gt_dir, split)) - image_infos = collect_annotations(files, nproc=args.nproc) - cvt_annotations(image_infos, osp.join(out_dir, json_name)) - - -if __name__ == "__main__": - main() diff --git a/tools/dataset_converters/images2coco.py b/tools/dataset_converters/images2coco.py deleted file mode 100644 index 35f7c1d6..00000000 --- a/tools/dataset_converters/images2coco.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import os - -import mmcv -from PIL import Image - - -def parse_args(): - parser = argparse.ArgumentParser(description="Convert images to coco format without annotations") - parser.add_argument("img_path", help="The root path of images") - parser.add_argument("classes", type=str, help="The text file name of storage class list") - parser.add_argument( - "out", - type=str, - help="The output annotation json file name, The save dir is in the same directory as img_path", - ) - parser.add_argument( - "-e", - "--exclude-extensions", - type=str, - nargs="+", - help='The suffix of images to be excluded, such as "png" and "bmp"', - ) - args = parser.parse_args() - return args - - -def collect_image_infos(path, exclude_extensions=None): - img_infos = [] - - images_generator = mmcv.scandir(path, recursive=True) - for image_path in mmcv.track_iter_progress(list(images_generator)): - if exclude_extensions is None or ( - exclude_extensions is not None and not image_path.lower().endswith(exclude_extensions) - ): - image_path = os.path.join(path, image_path) - img_pillow = Image.open(image_path) - img_info = { - "filename": image_path, - "width": img_pillow.width, - "height": img_pillow.height, - } - img_infos.append(img_info) - return img_infos - - -def cvt_to_coco_json(img_infos, classes): - image_id = 0 - coco = dict() - coco["images"] = [] - coco["type"] = "instance" - coco["categories"] = [] - coco["annotations"] = [] - image_set = set() - - for category_id, name in enumerate(classes): - category_item = dict() - category_item["supercategory"] = "none" - category_item["id"] = int(category_id) - category_item["name"] = str(name) - coco["categories"].append(category_item) - - for img_dict in img_infos: - file_name = img_dict["filename"] - assert file_name not in image_set - image_item = dict() - image_item["id"] = int(image_id) - image_item["file_name"] = str(file_name) - image_item["height"] = int(img_dict["height"]) - image_item["width"] = int(img_dict["width"]) - coco["images"].append(image_item) - image_set.add(file_name) - - image_id += 1 - return coco - - -def main(): - args = parse_args() - assert args.out.endswith("json"), "The output file name must be json suffix" - - # 1 load image list info - img_infos = collect_image_infos(args.img_path, args.exclude_extensions) - - # 2 convert to coco format data - classes = mmcv.list_from_file(args.classes) - coco_info = cvt_to_coco_json(img_infos, classes) - - # 3 dump - save_dir = os.path.join(args.img_path, "..", "annotations") - mmcv.mkdir_or_exist(save_dir) - save_path = os.path.join(save_dir, args.out) - mmcv.dump(coco_info, save_path) - print(f"save json file: {save_path}") - - -if __name__ == "__main__": - main() diff --git a/tools/dataset_converters/pascal_voc.py b/tools/dataset_converters/pascal_voc.py deleted file mode 100644 index ac025223..00000000 --- a/tools/dataset_converters/pascal_voc.py +++ /dev/null @@ -1,222 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import os.path as osp -import xml.etree.ElementTree as ET - -import mmcv -import numpy as np -from mmdet.core import voc_classes - -label_ids = {name: i for i, name in enumerate(voc_classes())} - - -def parse_xml(args): - xml_path, img_path = args - tree = ET.parse(xml_path) - root = tree.getroot() - size = root.find("size") - w = int(size.find("width").text) - h = int(size.find("height").text) - bboxes = [] - labels = [] - bboxes_ignore = [] - labels_ignore = [] - for obj in root.findall("object"): - name = obj.find("name").text - label = label_ids[name] - difficult = int(obj.find("difficult").text) - bnd_box = obj.find("bndbox") - bbox = [ - int(bnd_box.find("xmin").text), - int(bnd_box.find("ymin").text), - int(bnd_box.find("xmax").text), - int(bnd_box.find("ymax").text), - ] - if difficult: - bboxes_ignore.append(bbox) - labels_ignore.append(label) - else: - bboxes.append(bbox) - labels.append(label) - if not bboxes: - bboxes = np.zeros((0, 4)) - labels = np.zeros((0,)) - else: - bboxes = np.array(bboxes, ndmin=2) - 1 - labels = np.array(labels) - if not bboxes_ignore: - bboxes_ignore = np.zeros((0, 4)) - labels_ignore = np.zeros((0,)) - else: - bboxes_ignore = np.array(bboxes_ignore, ndmin=2) - 1 - labels_ignore = np.array(labels_ignore) - annotation = { - "filename": img_path, - "width": w, - "height": h, - "ann": { - "bboxes": bboxes.astype(np.float32), - "labels": labels.astype(np.int64), - "bboxes_ignore": bboxes_ignore.astype(np.float32), - "labels_ignore": labels_ignore.astype(np.int64), - }, - } - return annotation - - -def cvt_annotations(devkit_path, years, split, out_file): - if not isinstance(years, list): - years = [years] - annotations = [] - for year in years: - filelist = osp.join(devkit_path, f"VOC{year}/ImageSets/Main/{split}.txt") - if not osp.isfile(filelist): - print(f"filelist does not exist: {filelist}, skip voc{year} {split}") - return - img_names = mmcv.list_from_file(filelist) - xml_paths = [osp.join(devkit_path, f"VOC{year}/Annotations/{img_name}.xml") for img_name in img_names] - img_paths = [f"VOC{year}/JPEGImages/{img_name}.jpg" for img_name in img_names] - part_annotations = mmcv.track_progress(parse_xml, list(zip(xml_paths, img_paths))) - annotations.extend(part_annotations) - if out_file.endswith("json"): - annotations = cvt_to_coco_json(annotations) - mmcv.dump(annotations, out_file) - return annotations - - -def cvt_to_coco_json(annotations): - image_id = 0 - annotation_id = 0 - coco = dict() - coco["images"] = [] - coco["type"] = "instance" - coco["categories"] = [] - coco["annotations"] = [] - image_set = set() - - def addAnnItem(annotation_id, image_id, category_id, bbox, difficult_flag): - annotation_item = dict() - annotation_item["segmentation"] = [] - - seg = [] - # bbox[] is x1,y1,x2,y2 - # left_top - seg.append(int(bbox[0])) - seg.append(int(bbox[1])) - # left_bottom - seg.append(int(bbox[0])) - seg.append(int(bbox[3])) - # right_bottom - seg.append(int(bbox[2])) - seg.append(int(bbox[3])) - # right_top - seg.append(int(bbox[2])) - seg.append(int(bbox[1])) - - annotation_item["segmentation"].append(seg) - - xywh = np.array([bbox[0], bbox[1], bbox[2] - bbox[0], bbox[3] - bbox[1]]) - annotation_item["area"] = int(xywh[2] * xywh[3]) - if difficult_flag == 1: - annotation_item["ignore"] = 0 - annotation_item["iscrowd"] = 1 - else: - annotation_item["ignore"] = 0 - annotation_item["iscrowd"] = 0 - annotation_item["image_id"] = int(image_id) - annotation_item["bbox"] = xywh.astype(int).tolist() - annotation_item["category_id"] = int(category_id) - annotation_item["id"] = int(annotation_id) - coco["annotations"].append(annotation_item) - return annotation_id + 1 - - for category_id, name in enumerate(voc_classes()): - category_item = dict() - category_item["supercategory"] = "none" - category_item["id"] = int(category_id) - category_item["name"] = str(name) - coco["categories"].append(category_item) - - for ann_dict in annotations: - file_name = ann_dict["filename"] - ann = ann_dict["ann"] - assert file_name not in image_set - image_item = dict() - image_item["id"] = int(image_id) - image_item["file_name"] = str(file_name) - image_item["height"] = int(ann_dict["height"]) - image_item["width"] = int(ann_dict["width"]) - coco["images"].append(image_item) - image_set.add(file_name) - - bboxes = ann["bboxes"][:, :4] - labels = ann["labels"] - for bbox_id in range(len(bboxes)): - bbox = bboxes[bbox_id] - label = labels[bbox_id] - annotation_id = addAnnItem(annotation_id, image_id, label, bbox, difficult_flag=0) - - bboxes_ignore = ann["bboxes_ignore"][:, :4] - labels_ignore = ann["labels_ignore"] - for bbox_id in range(len(bboxes_ignore)): - bbox = bboxes_ignore[bbox_id] - label = labels_ignore[bbox_id] - annotation_id = addAnnItem(annotation_id, image_id, label, bbox, difficult_flag=1) - - image_id += 1 - - return coco - - -def parse_args(): - parser = argparse.ArgumentParser(description="Convert PASCAL VOC annotations to mmdetection format") - parser.add_argument("devkit_path", help="pascal voc devkit path") - parser.add_argument("-o", "--out-dir", help="output path") - parser.add_argument( - "--out-format", - default="pkl", - choices=("pkl", "coco"), - help='output format, "coco" indicates coco annotation format', - ) - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - devkit_path = args.devkit_path - out_dir = args.out_dir if args.out_dir else devkit_path - mmcv.mkdir_or_exist(out_dir) - - years = [] - if osp.isdir(osp.join(devkit_path, "VOC2007")): - years.append("2007") - if osp.isdir(osp.join(devkit_path, "VOC2012")): - years.append("2012") - if "2007" in years and "2012" in years: - years.append(["2007", "2012"]) - if not years: - raise OSError(f'The devkit path {devkit_path} contains neither "VOC2007" nor "VOC2012" subfolder') - out_fmt = f".{args.out_format}" - if args.out_format == "coco": - out_fmt = ".json" - for year in years: - if year == "2007": - prefix = "voc07" - elif year == "2012": - prefix = "voc12" - elif year == ["2007", "2012"]: - prefix = "voc0712" - for split in ["train", "val", "trainval"]: - dataset_name = prefix + "_" + split - print(f"processing {dataset_name} ...") - cvt_annotations(devkit_path, year, split, osp.join(out_dir, dataset_name + out_fmt)) - if not isinstance(year, list): - dataset_name = prefix + "_test" - print(f"processing {dataset_name} ...") - cvt_annotations(devkit_path, year, "test", osp.join(out_dir, dataset_name + out_fmt)) - print("Done!") - - -if __name__ == "__main__": - main() diff --git a/tools/deployment/mmdet2torchserve.py b/tools/deployment/mmdet2torchserve.py deleted file mode 100644 index 8cb3c6ca..00000000 --- a/tools/deployment/mmdet2torchserve.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -from argparse import ArgumentParser, Namespace -from pathlib import Path -from tempfile import TemporaryDirectory - -import mmcv - -try: - from model_archiver.model_packaging import package_model - from model_archiver.model_packaging_utils import ModelExportUtils -except ImportError: - package_model = None - - -def mmdet2torchserve( - config_file: str, - checkpoint_file: str, - output_folder: str, - model_name: str, - model_version: str = "1.0", - force: bool = False, -): - """Converts MMDetection model (config + checkpoint) to TorchServe `.mar`. - - Args: - config_file: - In MMDetection config format. - The contents vary for each task repository. - checkpoint_file: - In MMDetection checkpoint format. - The contents vary for each task repository. - output_folder: - Folder where `{model_name}.mar` will be created. - The file created will be in TorchServe archive format. - model_name: - If not None, used for naming the `{model_name}.mar` file - that will be created under `output_folder`. - If None, `{Path(checkpoint_file).stem}` will be used. - model_version: - Model's version. - force: - If True, if there is an existing `{model_name}.mar` - file under `output_folder` it will be overwritten. - """ - mmcv.mkdir_or_exist(output_folder) - - config = mmcv.Config.fromfile(config_file) - - with TemporaryDirectory() as tmpdir: - config.dump(f"{tmpdir}/config.py") - - args = Namespace( - **{ - "model_file": f"{tmpdir}/config.py", - "serialized_file": checkpoint_file, - "handler": f"{Path(__file__).parent}/mmdet_handler.py", - "model_name": model_name or Path(checkpoint_file).stem, - "version": model_version, - "export_path": output_folder, - "force": force, - "requirements_file": None, - "extra_files": None, - "runtime": "python", - "archive_format": "default", - } - ) - manifest = ModelExportUtils.generate_manifest_json(args) - package_model(args, manifest) - - -def parse_args(): - parser = ArgumentParser(description="Convert MMDetection models to TorchServe `.mar` format.") - parser.add_argument("config", type=str, help="config file path") - parser.add_argument("checkpoint", type=str, help="checkpoint file path") - parser.add_argument( - "--output-folder", - type=str, - required=True, - help="Folder where `{model_name}.mar` will be created.", - ) - parser.add_argument( - "--model-name", - type=str, - default=None, - help="If not None, used for naming the `{model_name}.mar`" - "file that will be created under `output_folder`." - "If None, `{Path(checkpoint_file).stem}` will be used.", - ) - parser.add_argument("--model-version", type=str, default="1.0", help="Number used for versioning.") - parser.add_argument( - "-f", - "--force", - action="store_true", - help="overwrite the existing `{model_name}.mar`", - ) - args = parser.parse_args() - - return args - - -if __name__ == "__main__": - args = parse_args() - - if package_model is None: - raise ImportError("`torch-model-archiver` is required.Try: pip install torch-model-archiver") - - mmdet2torchserve( - args.config, - args.checkpoint, - args.output_folder, - args.model_name, - args.model_version, - args.force, - ) diff --git a/tools/deployment/mmdet_handler.py b/tools/deployment/mmdet_handler.py deleted file mode 100644 index ce2ad099..00000000 --- a/tools/deployment/mmdet_handler.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import base64 -import os - -import mmcv -import torch -from mmdet.apis import inference_detector, init_detector -from ts.torch_handler.base_handler import BaseHandler - - -class MMdetHandler(BaseHandler): - threshold = 0.5 - - def initialize(self, context): - properties = context.system_properties - self.map_location = "cuda" if torch.cuda.is_available() else "cpu" - self.device = torch.device( - self.map_location + ":" + str(properties.get("gpu_id")) if torch.cuda.is_available() else self.map_location - ) - self.manifest = context.manifest - - model_dir = properties.get("model_dir") - serialized_file = self.manifest["model"]["serializedFile"] - checkpoint = os.path.join(model_dir, serialized_file) - self.config_file = os.path.join(model_dir, "config.py") - - self.model = init_detector(self.config_file, checkpoint, self.device) - self.initialized = True - - def preprocess(self, data): - images = [] - - for row in data: - image = row.get("data") or row.get("body") - if isinstance(image, str): - image = base64.b64decode(image) - image = mmcv.imfrombytes(image) - images.append(image) - - return images - - def inference(self, data, *args, **kwargs): - results = inference_detector(self.model, data) - return results - - def postprocess(self, data): - # Format output following the example ObjectDetectionHandler format - output = [] - for image_index, image_result in enumerate(data): - output.append([]) - if isinstance(image_result, tuple): - bbox_result, segm_result = image_result - if isinstance(segm_result, tuple): - segm_result = segm_result[0] # ms rcnn - else: - bbox_result, segm_result = image_result, None - - for class_index, class_result in enumerate(bbox_result): - class_name = self.model.CLASSES[class_index] - for bbox in class_result: - bbox_coords = bbox[:-1].tolist() - score = float(bbox[-1]) - if score >= self.threshold: - output[image_index].append( - { - "class_name": class_name, - "bbox": bbox_coords, - "score": score, - } - ) - - return output diff --git a/tools/deployment/onnx2tensorrt.py b/tools/deployment/onnx2tensorrt.py deleted file mode 100644 index 194ccfe3..00000000 --- a/tools/deployment/onnx2tensorrt.py +++ /dev/null @@ -1,273 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import os -import os.path as osp -import warnings - -import numpy as np -import onnx -import torch -from mmcv import Config -from mmcv.tensorrt import is_tensorrt_plugin_loaded, onnx2trt, save_trt_engine -from mmdet.core.export import preprocess_example_input -from mmdet.core.export.model_wrappers import ONNXRuntimeDetector, TensorRTDetector -from mmdet.datasets import DATASETS - - -def get_GiB(x: int): - """return x GiB.""" - return x * (1 << 30) - - -def onnx2tensorrt( - onnx_file, - trt_file, - input_config, - verify=False, - show=False, - workspace_size=1, - verbose=False, -): - import tensorrt as trt - - onnx_model = onnx.load(onnx_file) - max_shape = input_config["max_shape"] - min_shape = input_config["min_shape"] - opt_shape = input_config["opt_shape"] - fp16_mode = False - # create trt engine and wrapper - opt_shape_dict = {"input": [min_shape, opt_shape, max_shape]} - max_workspace_size = get_GiB(workspace_size) - trt_engine = onnx2trt( - onnx_model, - opt_shape_dict, - log_level=trt.Logger.VERBOSE if verbose else trt.Logger.ERROR, - fp16_mode=fp16_mode, - max_workspace_size=max_workspace_size, - ) - save_dir, _ = osp.split(trt_file) - if save_dir: - os.makedirs(save_dir, exist_ok=True) - save_trt_engine(trt_engine, trt_file) - print(f"Successfully created TensorRT engine: {trt_file}") - - if verify: - # prepare input - one_img, one_meta = preprocess_example_input(input_config) - img_list, img_meta_list = [one_img], [[one_meta]] - img_list = [_.cuda().contiguous() for _ in img_list] - - # wrap ONNX and TensorRT model - onnx_model = ONNXRuntimeDetector(onnx_file, CLASSES, device_id=0) - trt_model = TensorRTDetector(trt_file, CLASSES, device_id=0) - - # inference with wrapped model - with torch.no_grad(): - onnx_results = onnx_model(img_list, img_metas=img_meta_list, return_loss=False)[0] - trt_results = trt_model(img_list, img_metas=img_meta_list, return_loss=False)[0] - - if show: - out_file_ort, out_file_trt = None, None - else: - out_file_ort, out_file_trt = "show-ort.png", "show-trt.png" - show_img = one_meta["show_img"] - score_thr = 0.3 - onnx_model.show_result( - show_img, - onnx_results, - score_thr=score_thr, - show=True, - win_name="ONNXRuntime", - out_file=out_file_ort, - ) - trt_model.show_result( - show_img, - trt_results, - score_thr=score_thr, - show=True, - win_name="TensorRT", - out_file=out_file_trt, - ) - with_mask = trt_model.with_masks - # compare a part of result - if with_mask: - compare_pairs = list(zip(onnx_results, trt_results)) - else: - compare_pairs = [(onnx_results, trt_results)] - err_msg = ( - "The numerical values are different between Pytorch" - + " and ONNX, but it does not necessarily mean the" - + " exported ONNX model is problematic." - ) - # check the numerical value - for onnx_res, pytorch_res in compare_pairs: - for o_res, p_res in zip(onnx_res, pytorch_res): - np.testing.assert_allclose(o_res, p_res, rtol=1e-03, atol=1e-05, err_msg=err_msg) - print("The numerical values are the same between Pytorch and ONNX") - - -def parse_normalize_cfg(test_pipeline): - transforms = None - for pipeline in test_pipeline: - if "transforms" in pipeline: - transforms = pipeline["transforms"] - break - assert transforms is not None, "Failed to find `transforms`" - norm_config_li = [_ for _ in transforms if _["type"] == "Normalize"] - assert len(norm_config_li) == 1, "`norm_config` should only have one" - norm_config = norm_config_li[0] - return norm_config - - -def parse_args(): - parser = argparse.ArgumentParser(description="Convert MMDetection models from ONNX to TensorRT") - parser.add_argument("config", help="test config file path") - parser.add_argument("model", help="Filename of input ONNX model") - parser.add_argument( - "--trt-file", - type=str, - default="tmp.trt", - help="Filename of output TensorRT engine", - ) - parser.add_argument("--input-img", type=str, default="", help="Image for test") - parser.add_argument("--show", action="store_true", help="Whether to show output results") - parser.add_argument( - "--dataset", - type=str, - default="coco", - help="Dataset name. This argument is deprecated and will be \ - removed in future releases.", - ) - parser.add_argument( - "--verify", - action="store_true", - help="Verify the outputs of ONNXRuntime and TensorRT", - ) - parser.add_argument( - "--verbose", - action="store_true", - help="Whether to verbose logging messages while creating \ - TensorRT engine. Defaults to False.", - ) - parser.add_argument( - "--to-rgb", - action="store_false", - help="Feed model with RGB or BGR image. Default is RGB. This \ - argument is deprecated and will be removed in future releases.", - ) - parser.add_argument( - "--shape", - type=int, - nargs="+", - default=[400, 600], - help="Input size of the model", - ) - parser.add_argument( - "--mean", - type=float, - nargs="+", - default=[123.675, 116.28, 103.53], - help="Mean value used for preprocess input data. This argument \ - is deprecated and will be removed in future releases.", - ) - parser.add_argument( - "--std", - type=float, - nargs="+", - default=[58.395, 57.12, 57.375], - help="Variance value used for preprocess input data. \ - This argument is deprecated and will be removed in future releases.", - ) - parser.add_argument( - "--min-shape", - type=int, - nargs="+", - default=None, - help="Minimum input size of the model in TensorRT", - ) - parser.add_argument( - "--max-shape", - type=int, - nargs="+", - default=None, - help="Maximum input size of the model in TensorRT", - ) - parser.add_argument("--workspace-size", type=int, default=1, help="Max workspace size in GiB") - - args = parser.parse_args() - return args - - -if __name__ == "__main__": - assert is_tensorrt_plugin_loaded(), "TensorRT plugin should be compiled." - args = parse_args() - warnings.warn( - "Arguments like `--to-rgb`, `--mean`, `--std`, `--dataset` would be \ - parsed directly from config file and are deprecated and will be \ - removed in future releases." - ) - if not args.input_img: - args.input_img = osp.join(osp.dirname(__file__), "../../demo/demo.jpg") - - cfg = Config.fromfile(args.config) - - def parse_shape(shape): - if len(shape) == 1: - shape = (1, 3, shape[0], shape[0]) - elif len(args.shape) == 2: - shape = (1, 3) + tuple(shape) - else: - raise ValueError("invalid input shape") - return shape - - if args.shape: - input_shape = parse_shape(args.shape) - else: - img_scale = cfg.test_pipeline[1]["img_scale"] - input_shape = (1, 3, img_scale[1], img_scale[0]) - - if not args.max_shape: - max_shape = input_shape - else: - max_shape = parse_shape(args.max_shape) - - if not args.min_shape: - min_shape = input_shape - else: - min_shape = parse_shape(args.min_shape) - - dataset = DATASETS.get(cfg.data.test["type"]) - assert dataset is not None - CLASSES = dataset.CLASSES - normalize_cfg = parse_normalize_cfg(cfg.test_pipeline) - - input_config = { - "min_shape": min_shape, - "opt_shape": input_shape, - "max_shape": max_shape, - "input_shape": input_shape, - "input_path": args.input_img, - "normalize_cfg": normalize_cfg, - } - # Create TensorRT engine - onnx2tensorrt( - args.model, - args.trt_file, - input_config, - verify=args.verify, - show=args.show, - workspace_size=args.workspace_size, - verbose=args.verbose, - ) - - # Following strings of text style are from colorama package - bright_style, reset_style = "\x1b[1m", "\x1b[0m" - red_text, blue_text = "\x1b[31m", "\x1b[34m" - white_background = "\x1b[107m" - - msg = white_background + bright_style + red_text - msg += "DeprecationWarning: This tool will be deprecated in future. " - msg += blue_text + "Welcome to use the unified model deployment toolbox " - msg += "MMDeploy: https://github.com/open-mmlab/mmdeploy" - msg += reset_style - warnings.warn(msg) diff --git a/tools/deployment/pytorch2onnx.py b/tools/deployment/pytorch2onnx.py deleted file mode 100644 index 29cfb938..00000000 --- a/tools/deployment/pytorch2onnx.py +++ /dev/null @@ -1,328 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import os.path as osp -import warnings -from functools import partial - -import numpy as np -import onnx -import torch -from mmcv import Config, DictAction -from mmdet.core.export import build_model_from_cfg, preprocess_example_input -from mmdet.core.export.model_wrappers import ONNXRuntimeDetector - - -def pytorch2onnx( - model, - input_img, - input_shape, - normalize_cfg, - opset_version=11, - show=False, - output_file="tmp.onnx", - verify=False, - test_img=None, - do_simplify=False, - dynamic_export=None, - skip_postprocess=False, -): - input_config = { - "input_shape": input_shape, - "input_path": input_img, - "normalize_cfg": normalize_cfg, - } - # prepare input - one_img, one_meta = preprocess_example_input(input_config) - img_list, img_meta_list = [one_img], [[one_meta]] - - if skip_postprocess: - warnings.warn("Not all models support export onnx without post process, especially two stage detectors!") - model.forward = model.forward_dummy - torch.onnx.export( - model, - one_img, - output_file, - input_names=["input"], - export_params=True, - keep_initializers_as_inputs=True, - do_constant_folding=True, - verbose=show, - opset_version=opset_version, - ) - - print(f"Successfully exported ONNX model without post process: {output_file}") - return - - # replace original forward function - origin_forward = model.forward - model.forward = partial(model.forward, img_metas=img_meta_list, return_loss=False, rescale=False) - - output_names = ["dets", "labels"] - if model.with_mask: - output_names.append("masks") - input_name = "input" - dynamic_axes = None - if dynamic_export: - dynamic_axes = { - input_name: {0: "batch", 2: "height", 3: "width"}, - "dets": { - 0: "batch", - 1: "num_dets", - }, - "labels": { - 0: "batch", - 1: "num_dets", - }, - } - if model.with_mask: - dynamic_axes["masks"] = {0: "batch", 1: "num_dets"} - - torch.onnx.export( - model, - img_list, - output_file, - input_names=[input_name], - output_names=output_names, - export_params=True, - keep_initializers_as_inputs=True, - do_constant_folding=True, - verbose=show, - opset_version=opset_version, - dynamic_axes=dynamic_axes, - ) - - model.forward = origin_forward - - if do_simplify: - import onnxsim - from mmdet import digit_version - - min_required_version = "0.4.0" - assert digit_version(onnxsim.__version__) >= digit_version(min_required_version), ( - f"Requires to install onnxsim>={min_required_version}" - ) - - model_opt, check_ok = onnxsim.simplify(output_file) - if check_ok: - onnx.save(model_opt, output_file) - print(f"Successfully simplified ONNX model: {output_file}") - else: - warnings.warn("Failed to simplify ONNX model.") - print(f"Successfully exported ONNX model: {output_file}") - - if verify: - # check by onnx - onnx_model = onnx.load(output_file) - onnx.checker.check_model(onnx_model) - - # wrap onnx model - onnx_model = ONNXRuntimeDetector(output_file, model.CLASSES, 0) - if dynamic_export: - # scale up to test dynamic shape - h, w = [int((_ * 1.5) // 32 * 32) for _ in input_shape[2:]] - h, w = min(1344, h), min(1344, w) - input_config["input_shape"] = (1, 3, h, w) - - if test_img is None: - input_config["input_path"] = input_img - - # prepare input once again - one_img, one_meta = preprocess_example_input(input_config) - img_list, img_meta_list = [one_img], [[one_meta]] - - # get pytorch output - with torch.no_grad(): - pytorch_results = model(img_list, img_metas=img_meta_list, return_loss=False, rescale=True)[0] - - img_list = [_.cuda().contiguous() for _ in img_list] - if dynamic_export: - img_list = img_list + [_.flip(-1).contiguous() for _ in img_list] - img_meta_list = img_meta_list * 2 - # get onnx output - onnx_results = onnx_model(img_list, img_metas=img_meta_list, return_loss=False)[0] - # visualize predictions - score_thr = 0.3 - if show: - out_file_ort, out_file_pt = None, None - else: - out_file_ort, out_file_pt = "show-ort.png", "show-pt.png" - - show_img = one_meta["show_img"] - model.show_result( - show_img, - pytorch_results, - score_thr=score_thr, - show=True, - win_name="PyTorch", - out_file=out_file_pt, - ) - onnx_model.show_result( - show_img, - onnx_results, - score_thr=score_thr, - show=True, - win_name="ONNXRuntime", - out_file=out_file_ort, - ) - - # compare a part of result - if model.with_mask: - compare_pairs = list(zip(onnx_results, pytorch_results)) - else: - compare_pairs = [(onnx_results, pytorch_results)] - err_msg = ( - "The numerical values are different between Pytorch" - + " and ONNX, but it does not necessarily mean the" - + " exported ONNX model is problematic." - ) - # check the numerical value - for onnx_res, pytorch_res in compare_pairs: - for o_res, p_res in zip(onnx_res, pytorch_res): - np.testing.assert_allclose(o_res, p_res, rtol=1e-03, atol=1e-05, err_msg=err_msg) - print("The numerical values are the same between Pytorch and ONNX") - - -def parse_normalize_cfg(test_pipeline): - transforms = None - for pipeline in test_pipeline: - if "transforms" in pipeline: - transforms = pipeline["transforms"] - break - assert transforms is not None, "Failed to find `transforms`" - norm_config_li = [_ for _ in transforms if _["type"] == "Normalize"] - assert len(norm_config_li) == 1, "`norm_config` should only have one" - norm_config = norm_config_li[0] - return norm_config - - -def parse_args(): - parser = argparse.ArgumentParser(description="Convert MMDetection models to ONNX") - parser.add_argument("config", help="test config file path") - parser.add_argument("checkpoint", help="checkpoint file") - parser.add_argument("--input-img", type=str, help="Images for input") - parser.add_argument("--show", action="store_true", help="Show onnx graph and detection outputs") - parser.add_argument("--output-file", type=str, default="tmp.onnx") - parser.add_argument("--opset-version", type=int, default=11) - parser.add_argument("--test-img", type=str, default=None, help="Images for test") - parser.add_argument( - "--dataset", - type=str, - default="coco", - help="Dataset name. This argument is deprecated and will be removed \ - in future releases.", - ) - parser.add_argument( - "--verify", - action="store_true", - help="verify the onnx model output against pytorch output", - ) - parser.add_argument("--simplify", action="store_true", help="Whether to simplify onnx model.") - parser.add_argument("--shape", type=int, nargs="+", default=[800, 1216], help="input image size") - parser.add_argument( - "--mean", - type=float, - nargs="+", - default=[123.675, 116.28, 103.53], - help="mean value used for preprocess input data.This argument \ - is deprecated and will be removed in future releases.", - ) - parser.add_argument( - "--std", - type=float, - nargs="+", - default=[58.395, 57.12, 57.375], - help="variance value used for preprocess input data. " - "This argument is deprecated and will be removed in future releases.", - ) - parser.add_argument( - "--cfg-options", - nargs="+", - action=DictAction, - help="Override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file. If the value to " - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - "Note that the quotation marks are necessary and that no white space " - "is allowed.", - ) - parser.add_argument( - "--dynamic-export", - action="store_true", - help="Whether to export onnx with dynamic axis.", - ) - parser.add_argument( - "--skip-postprocess", - action="store_true", - help="Whether to export model without post process. Experimental " - "option. We do not guarantee the correctness of the exported " - "model.", - ) - args = parser.parse_args() - return args - - -if __name__ == "__main__": - args = parse_args() - warnings.warn( - "Arguments like `--mean`, `--std`, `--dataset` would be \ - parsed directly from config file and are deprecated and \ - will be removed in future releases." - ) - - assert args.opset_version == 11, "MMDet only support opset 11 now" - - try: - from mmcv.onnx.symbolic import register_extra_symbolics - except ModuleNotFoundError: - raise NotImplementedError("please update mmcv to version>=v1.0.4") - register_extra_symbolics(args.opset_version) - - cfg = Config.fromfile(args.config) - if args.cfg_options is not None: - cfg.merge_from_dict(args.cfg_options) - - if args.shape is None: - img_scale = cfg.test_pipeline[1]["img_scale"] - input_shape = (1, 3, img_scale[1], img_scale[0]) - elif len(args.shape) == 1: - input_shape = (1, 3, args.shape[0], args.shape[0]) - elif len(args.shape) == 2: - input_shape = (1, 3) + tuple(args.shape) - else: - raise ValueError("invalid input shape") - - # build the model and load checkpoint - model = build_model_from_cfg(args.config, args.checkpoint, args.cfg_options) - - if not args.input_img: - args.input_img = osp.join(osp.dirname(__file__), "../../demo/demo.jpg") - - normalize_cfg = parse_normalize_cfg(cfg.test_pipeline) - - # convert model to onnx file - pytorch2onnx( - model, - args.input_img, - input_shape, - normalize_cfg, - opset_version=args.opset_version, - show=args.show, - output_file=args.output_file, - verify=args.verify, - test_img=args.test_img, - do_simplify=args.simplify, - dynamic_export=args.dynamic_export, - skip_postprocess=args.skip_postprocess, - ) - - # Following strings of text style are from colorama package - bright_style, reset_style = "\x1b[1m", "\x1b[0m" - red_text, blue_text = "\x1b[31m", "\x1b[34m" - white_background = "\x1b[107m" - - msg = white_background + bright_style + red_text - msg += "DeprecationWarning: This tool will be deprecated in future. " - msg += blue_text + "Welcome to use the unified model deployment toolbox " - msg += "MMDeploy: https://github.com/open-mmlab/mmdeploy" - msg += reset_style - warnings.warn(msg) diff --git a/tools/deployment/test.py b/tools/deployment/test.py deleted file mode 100644 index d3b82c66..00000000 --- a/tools/deployment/test.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import warnings - -import mmcv -from mmcv import Config, DictAction -from mmcv.parallel import MMDataParallel -from mmdet.apis import single_gpu_test -from mmdet.datasets import build_dataloader, build_dataset, replace_ImageToTensor -from mmdet.utils import compat_cfg - - -def parse_args(): - parser = argparse.ArgumentParser(description="MMDet test (and eval) an ONNX model using ONNXRuntime") - parser.add_argument("config", help="test config file path") - parser.add_argument("model", help="Input model file") - parser.add_argument("--out", help="output result file in pickle format") - parser.add_argument( - "--format-only", - action="store_true", - help="Format the output results without perform evaluation. It is" - "useful when you want to format the result to a specific format and " - "submit it to the test server", - ) - parser.add_argument( - "--backend", - required=True, - choices=["onnxruntime", "tensorrt"], - help="Backend for input model to run. ", - ) - parser.add_argument( - "--eval", - type=str, - nargs="+", - help='evaluation metrics, which depends on the dataset, e.g., "bbox",' - ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC', - ) - parser.add_argument("--show", action="store_true", help="show results") - parser.add_argument("--show-dir", help="directory where painted images will be saved") - parser.add_argument( - "--show-score-thr", - type=float, - default=0.3, - help="score threshold (default: 0.3)", - ) - parser.add_argument( - "--cfg-options", - nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file. If the value to " - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - "Note that the quotation marks are necessary and that no white space " - "is allowed.", - ) - parser.add_argument( - "--eval-options", - nargs="+", - action=DictAction, - help="custom options for evaluation, the key-value pair in xxx=yyy " - "format will be kwargs for dataset.evaluate() function", - ) - - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - - assert args.out or args.eval or args.format_only or args.show or args.show_dir, ( - "Please specify at least one operation (save/eval/format/show the " - 'results / save the results) with the argument "--out", "--eval"' - ', "--format-only", "--show" or "--show-dir"' - ) - - if args.eval and args.format_only: - raise ValueError("--eval and --format_only cannot be both specified") - - if args.out is not None and not args.out.endswith((".pkl", ".pickle")): - raise ValueError("The output file must be a pkl file.") - - cfg = Config.fromfile(args.config) - if args.cfg_options is not None: - cfg.merge_from_dict(args.cfg_options) - cfg = compat_cfg(cfg) - # in case the test dataset is concatenated - samples_per_gpu = 1 - if isinstance(cfg.data.test, dict): - cfg.data.test.test_mode = True - samples_per_gpu = cfg.data.test.pop("samples_per_gpu", 1) - if samples_per_gpu > 1: - # Replace 'ImageToTensor' to 'DefaultFormatBundle' - cfg.data.test.pipeline = replace_ImageToTensor(cfg.data.test.pipeline) - elif isinstance(cfg.data.test, list): - for ds_cfg in cfg.data.test: - ds_cfg.test_mode = True - samples_per_gpu = max([ds_cfg.pop("samples_per_gpu", 1) for ds_cfg in cfg.data.test]) - if samples_per_gpu > 1: - for ds_cfg in cfg.data.test: - ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) - - # build the dataloader - dataset = build_dataset(cfg.data.test) - data_loader = build_dataloader( - dataset, - samples_per_gpu=samples_per_gpu, - workers_per_gpu=cfg.data.workers_per_gpu, - dist=False, - shuffle=False, - ) - - if args.backend == "onnxruntime": - from mmdet.core.export.model_wrappers import ONNXRuntimeDetector - - model = ONNXRuntimeDetector(args.model, class_names=dataset.CLASSES, device_id=0) - elif args.backend == "tensorrt": - from mmdet.core.export.model_wrappers import TensorRTDetector - - model = TensorRTDetector(args.model, class_names=dataset.CLASSES, device_id=0) - - model = MMDataParallel(model, device_ids=[0]) - outputs = single_gpu_test(model, data_loader, args.show, args.show_dir, args.show_score_thr) - - if args.out: - print(f"\nwriting results to {args.out}") - mmcv.dump(outputs, args.out) - kwargs = {} if args.eval_options is None else args.eval_options - if args.format_only: - dataset.format_results(outputs, **kwargs) - if args.eval: - eval_kwargs = cfg.get("evaluation", {}).copy() - # hard-code way to remove EvalHook args - for key in ["interval", "tmpdir", "start", "gpu_collect", "save_best", "rule"]: - eval_kwargs.pop(key, None) - eval_kwargs.update(dict(metric=args.eval, **kwargs)) - print(dataset.evaluate(outputs, **eval_kwargs)) - - -if __name__ == "__main__": - main() - - # Following strings of text style are from colorama package - bright_style, reset_style = "\x1b[1m", "\x1b[0m" - red_text, blue_text = "\x1b[31m", "\x1b[34m" - white_background = "\x1b[107m" - - msg = white_background + bright_style + red_text - msg += "DeprecationWarning: This tool will be deprecated in future. " - msg += blue_text + "Welcome to use the unified model deployment toolbox " - msg += "MMDeploy: https://github.com/open-mmlab/mmdeploy" - msg += reset_style - warnings.warn(msg) diff --git a/tools/misc/.ipynb_checkpoints/browse_dataset-checkpoint.py b/tools/misc/.ipynb_checkpoints/browse_dataset-checkpoint.py deleted file mode 100644 index d9fb2851..00000000 --- a/tools/misc/.ipynb_checkpoints/browse_dataset-checkpoint.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import os -from collections import Sequence -from pathlib import Path - -import mmcv -import numpy as np -from mmcv import Config, DictAction - -from mmdet.core.utils import mask2ndarray -from mmdet.core.visualization import imshow_det_bboxes -from mmdet.datasets.builder import build_dataset -from mmdet.utils import replace_cfg_vals, update_data_root - - -def parse_args(): - parser = argparse.ArgumentParser(description='Browse a dataset') - parser.add_argument('config', help='train config file path') - parser.add_argument( - '--skip-type', - type=str, - nargs='+', - default=['DefaultFormatBundle', 'Normalize', 'Collect'], - help='skip some useless pipeline') - parser.add_argument( - '--output-dir', - default=None, - type=str, - help='If there is no display interface, you can save it') - parser.add_argument('--not-show', default=False, action='store_true') - parser.add_argument( - '--show-interval', - type=float, - default=2, - help='the interval of show (s)') - parser.add_argument( - '--cfg-options', - nargs='+', - action=DictAction, - help='override some settings in the used config, the key-value pair ' - 'in xxx=yyy format will be merged into config file. If the value to ' - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - 'Note that the quotation marks are necessary and that no white space ' - 'is allowed.') - args = parser.parse_args() - return args - - -def retrieve_data_cfg(config_path, skip_type, cfg_options): - - def skip_pipeline_steps(config): - config['pipeline'] = [ - x for x in config.pipeline if x['type'] not in skip_type - ] - - cfg = Config.fromfile(config_path) - - # replace the ${key} with the value of cfg.key - cfg = replace_cfg_vals(cfg) - - # update data root according to MMDET_DATASETS - update_data_root(cfg) - - if cfg_options is not None: - cfg.merge_from_dict(cfg_options) - train_data_cfg = cfg.data.train - while 'dataset' in train_data_cfg and train_data_cfg[ - 'type'] != 'MultiImageMixDataset': - train_data_cfg = train_data_cfg['dataset'] - - if isinstance(train_data_cfg, Sequence): - [skip_pipeline_steps(c) for c in train_data_cfg] - else: - skip_pipeline_steps(train_data_cfg) - - return cfg - - -def main(): - args = parse_args() - cfg = retrieve_data_cfg(args.config, args.skip_type, args.cfg_options) - - if 'gt_semantic_seg' in cfg.train_pipeline[-1]['keys']: - cfg.data.train.pipeline = [ - p for p in cfg.data.train.pipeline if p['type'] != 'SegRescale' - ] - dataset = build_dataset(cfg.data.train) - - progress_bar = mmcv.ProgressBar(len(dataset)) - - for item in dataset: - filename = os.path.join(args.output_dir, - Path(item['filename']).name - ) if args.output_dir is not None else None - - gt_bboxes = item['gt_bboxes'] - gt_labels = item['gt_labels'] - gt_masks = item.get('gt_masks', None) - if gt_masks is not None: - gt_masks = mask2ndarray(gt_masks) - - gt_seg = item.get('gt_semantic_seg', None) - if gt_seg is not None: - pad_value = 255 # the padding value of gt_seg - sem_labels = np.unique(gt_seg) - all_labels = np.concatenate((gt_labels, sem_labels), axis=0) - all_labels, counts = np.unique(all_labels, return_counts=True) - stuff_labels = all_labels[np.logical_and(counts < 2, - all_labels != pad_value)] - stuff_masks = gt_seg[None] == stuff_labels[:, None, None] - gt_labels = np.concatenate((gt_labels, stuff_labels), axis=0) - gt_masks = np.concatenate((gt_masks, stuff_masks.astype(np.uint8)), - axis=0) - # If you need to show the bounding boxes, - # please comment the following line - gt_bboxes = None - - imshow_det_bboxes( - item['img'], - gt_bboxes, - gt_labels, - gt_masks, - class_names=dataset.CLASSES, - show=not args.not_show, - wait_time=args.show_interval, - out_file=filename, - bbox_color=dataset.PALETTE, - text_color=(200, 200, 200), - mask_color=dataset.PALETTE) - - progress_bar.update() - - -if __name__ == '__main__': - main() diff --git a/tools/misc/browse_dataset.py b/tools/misc/browse_dataset.py deleted file mode 100644 index 5c1c70b5..00000000 --- a/tools/misc/browse_dataset.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import os -from collections.abc import Sequence -from pathlib import Path - -import mmcv -import numpy as np -from mmcv import Config, DictAction -from mmdet.core.utils import mask2ndarray -from mmdet.core.visualization import imshow_det_bboxes -from mmdet.datasets.builder import build_dataset -from mmdet.utils import replace_cfg_vals, update_data_root - - -def parse_args(): - parser = argparse.ArgumentParser(description="Browse a dataset") - parser.add_argument("config", help="train config file path") - parser.add_argument( - "--skip-type", - type=str, - nargs="+", - default=["DefaultFormatBundle", "Normalize", "Collect"], - help="skip some useless pipeline", - ) - parser.add_argument( - "--output-dir", - default=None, - type=str, - help="If there is no display interface, you can save it", - ) - parser.add_argument("--not-show", default=False, action="store_true") - parser.add_argument("--show-interval", type=float, default=2, help="the interval of show (s)") - parser.add_argument( - "--cfg-options", - nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file. If the value to " - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - "Note that the quotation marks are necessary and that no white space " - "is allowed.", - ) - args = parser.parse_args() - return args - - -def retrieve_data_cfg(config_path, skip_type, cfg_options): - def skip_pipeline_steps(config): - config["pipeline"] = [x for x in config.pipeline if x["type"] not in skip_type] - - cfg = Config.fromfile(config_path) - - # replace the ${key} with the value of cfg.key - cfg = replace_cfg_vals(cfg) - - # update data root according to MMDET_DATASETS - update_data_root(cfg) - - if cfg_options is not None: - cfg.merge_from_dict(cfg_options) - train_data_cfg = cfg.data.train - while "dataset" in train_data_cfg and train_data_cfg["type"] != "MultiImageMixDataset": - train_data_cfg = train_data_cfg["dataset"] - - if isinstance(train_data_cfg, Sequence): - [skip_pipeline_steps(c) for c in train_data_cfg] - else: - skip_pipeline_steps(train_data_cfg) - - return cfg - - -def main(): - args = parse_args() - cfg = retrieve_data_cfg(args.config, args.skip_type, args.cfg_options) - - if "gt_semantic_seg" in cfg.train_pipeline[-1]["keys"]: - cfg.data.train.pipeline = [p for p in cfg.data.train.pipeline if p["type"] != "SegRescale"] - dataset = build_dataset(cfg.data.train) - - progress_bar = mmcv.ProgressBar(len(dataset)) - - for item in dataset: - filename = os.path.join(args.output_dir, Path(item["filename"]).name) if args.output_dir is not None else None - - gt_bboxes = item["gt_bboxes"] - gt_labels = item["gt_labels"] - gt_masks = item.get("gt_masks", None) - if gt_masks is not None: - gt_masks = mask2ndarray(gt_masks) - - gt_seg = item.get("gt_semantic_seg", None) - if gt_seg is not None: - pad_value = 255 # the padding value of gt_seg - sem_labels = np.unique(gt_seg) - all_labels = np.concatenate((gt_labels, sem_labels), axis=0) - all_labels, counts = np.unique(all_labels, return_counts=True) - stuff_labels = all_labels[np.logical_and(counts < 2, all_labels != pad_value)] - stuff_masks = gt_seg[None] == stuff_labels[:, None, None] - gt_labels = np.concatenate((gt_labels, stuff_labels), axis=0) - gt_masks = np.concatenate((gt_masks, stuff_masks.astype(np.uint8)), axis=0) - # If you need to show the bounding boxes, - # please comment the following line - gt_bboxes = None - - imshow_det_bboxes( - item["img"], - gt_bboxes, - gt_labels, - gt_masks, - class_names=dataset.CLASSES, - show=not args.not_show, - wait_time=args.show_interval, - out_file=filename, - bbox_color=dataset.PALETTE, - text_color=(200, 200, 200), - mask_color=dataset.PALETTE, - ) - - progress_bar.update() - - -if __name__ == "__main__": - main() diff --git a/tools/misc/download_dataset.py b/tools/misc/download_dataset.py deleted file mode 100644 index defe0e00..00000000 --- a/tools/misc/download_dataset.py +++ /dev/null @@ -1,167 +0,0 @@ -import argparse -import tarfile -from itertools import repeat -from multiprocessing.pool import ThreadPool -from pathlib import Path -from tarfile import TarFile -from zipfile import ZipFile - -import torch -from mmcv.utils.path import mkdir_or_exist - - -def parse_args(): - parser = argparse.ArgumentParser(description="Download datasets for training") - parser.add_argument("--dataset-name", type=str, help="dataset name", default="coco2017") - parser.add_argument("--save-dir", type=str, help="the dir to save dataset", default="data/coco") - parser.add_argument( - "--unzip", - action="store_true", - help="whether unzip dataset or not, zipped files will be saved", - ) - parser.add_argument("--delete", action="store_true", help="delete the download zipped files") - parser.add_argument("--threads", type=int, help="number of threading", default=4) - args = parser.parse_args() - return args - - -def download(url, dir, unzip=True, delete=False, threads=1): - def download_one(url, dir): - f = dir / Path(url).name - if Path(url).is_file(): - Path(url).rename(f) - elif not f.exists(): - print(f"Downloading {url} to {f}") - torch.hub.download_url_to_file(url, f, progress=True) - if unzip and f.suffix in (".zip", ".tar"): - print(f"Unzipping {f.name}") - if f.suffix == ".zip": - ZipFile(f).extractall(path=dir) - elif f.suffix == ".tar": - TarFile(f).extractall(path=dir) - if delete: - f.unlink() - print(f"Delete {f}") - - dir = Path(dir) - if threads > 1: - pool = ThreadPool(threads) - pool.imap(lambda x: download_one(*x), zip(url, repeat(dir))) - pool.close() - pool.join() - else: - for u in [url] if isinstance(url, (str, Path)) else url: - download_one(u, dir) - - -def download_objects365v2(url, dir, unzip=True, delete=False, threads=1): - def download_single(url, dir): - if "train" in url: - saving_dir = dir / Path("train_zip") - mkdir_or_exist(saving_dir) - f = saving_dir / Path(url).name - - unzip_dir = dir / Path("train") - mkdir_or_exist(unzip_dir) - elif "val" in url: - saving_dir = dir / Path("val") - mkdir_or_exist(saving_dir) - f = saving_dir / Path(url).name - - unzip_dir = dir / Path("val") - mkdir_or_exist(unzip_dir) - else: - raise NotImplementedError - - if Path(url).is_file(): - Path(url).rename(f) - elif not f.exists(): - print(f"Downloading {url} to {f}") - torch.hub.download_url_to_file(url, f, progress=True) - - if unzip and str(f).endswith(".tar.gz"): - print(f"Unzipping {f.name}") - tar = tarfile.open(f) - tar.extractall(path=unzip_dir) - if delete: - f.unlink() - print(f"Delete {f}") - - # process annotations - full_url = [] - for _url in url: - if "zhiyuan_objv2_train.tar.gz" in _url or "zhiyuan_objv2_val.json" in _url: - full_url.append(_url) - elif "train" in _url: - for i in range(51): - full_url.append(f"{_url}patch{i}.tar.gz") - elif "val/images/v1" in _url: - for i in range(16): - full_url.append(f"{_url}patch{i}.tar.gz") - elif "val/images/v2" in _url: - for i in range(16, 44): - full_url.append(f"{_url}patch{i}.tar.gz") - else: - raise NotImplementedError - - dir = Path(dir) - if threads > 1: - pool = ThreadPool(threads) - pool.imap(lambda x: download_single(*x), zip(full_url, repeat(dir))) - pool.close() - pool.join() - else: - for u in full_url: - download_single(u, dir) - - -def main(): - args = parse_args() - path = Path(args.save_dir) - if not path.exists(): - path.mkdir(parents=True, exist_ok=True) - data2url = dict( - # TODO: Support for downloading Panoptic Segmentation of COCO - coco2017=[ - "http://images.cocodataset.org/zips/train2017.zip", - "http://images.cocodataset.org/zips/val2017.zip", - "http://images.cocodataset.org/zips/test2017.zip", - "http://images.cocodataset.org/annotations/" + "annotations_trainval2017.zip", - ], - lvis=[ - "https://s3-us-west-2.amazonaws.com/dl.fbaipublicfiles.com/LVIS/lvis_v1_train.json.zip", # noqa - "https://s3-us-west-2.amazonaws.com/dl.fbaipublicfiles.com/LVIS/lvis_v1_train.json.zip", # noqa - ], - voc2007=[ - "http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar", # noqa - "http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar", # noqa - "http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCdevkit_08-Jun-2007.tar", # noqa - ], - # Note: There is no download link for Objects365-V1 right now. If you - # would like to download Objects365-V1, please visit - # http://www.objects365.org/ to concat the author. - objects365v2=[ - # training annotations - "https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/train/zhiyuan_objv2_train.tar.gz", # noqa - # validation annotations - "https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/val/zhiyuan_objv2_val.json", # noqa - # training url root - "https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/train/", # noqa - # validation url root_1 - "https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/val/images/v1/", # noqa - # validation url root_2 - "https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/val/images/v2/", # noqa - ], - ) - url = data2url.get(args.dataset_name, None) - if url is None: - print("Only support COCO, VOC, LVIS, and Objects365v2 now!") - return - if args.dataset_name == "objects365v2": - download_objects365v2(url, dir=path, unzip=args.unzip, delete=args.delete, threads=args.threads) - else: - download(url, dir=path, unzip=args.unzip, delete=args.delete, threads=args.threads) - - -if __name__ == "__main__": - main() diff --git a/tools/misc/gen_coco_panoptic_test_info.py b/tools/misc/gen_coco_panoptic_test_info.py deleted file mode 100644 index 12c44119..00000000 --- a/tools/misc/gen_coco_panoptic_test_info.py +++ /dev/null @@ -1,30 +0,0 @@ -import argparse -import os.path as osp - -import mmcv - - -def parse_args(): - parser = argparse.ArgumentParser(description="Generate COCO test image information for COCO panoptic segmentation.") - parser.add_argument("data_root", help="Path to COCO annotation directory.") - args = parser.parse_args() - - return args - - -def main(): - args = parse_args() - data_root = args.data_root - val_info = mmcv.load(osp.join(data_root, "panoptic_val2017.json")) - test_old_info = mmcv.load(osp.join(data_root, "image_info_test-dev2017.json")) - - # replace categories from image_info_test-dev2017.json - # with categories from panoptic_val2017.json which - # has attribute `isthing`. - test_info = test_old_info - test_info.update({"categories": val_info["categories"]}) - mmcv.dump(test_info, osp.join(data_root, "panoptic_image_info_test-dev2017.json")) - - -if __name__ == "__main__": - main() diff --git a/tools/misc/get_image_metas.py b/tools/misc/get_image_metas.py deleted file mode 100644 index 2802e446..00000000 --- a/tools/misc/get_image_metas.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -"""Get test image metas on a specific dataset. - -Here is an example to run this script. - -Example: - python tools/misc/get_image_metas.py ${CONFIG} \ - --out ${OUTPUT FILE NAME} -""" - -import argparse -import csv -import os.path as osp -from multiprocessing import Pool - -import mmcv -from mmcv import Config - - -def parse_args(): - parser = argparse.ArgumentParser(description="Collect image metas") - parser.add_argument("config", help="Config file path") - parser.add_argument( - "--out", - default="validation-image-metas.pkl", - help="The output image metas file name. The save dir is in the same directory as `dataset.ann_file` path", - ) - parser.add_argument("--nproc", default=4, type=int, help="Processes used for get image metas") - args = parser.parse_args() - return args - - -def get_metas_from_csv_style_ann_file(ann_file): - data_infos = [] - cp_filename = None - with open(ann_file) as f: - reader = csv.reader(f) - for i, line in enumerate(reader): - if i == 0: - continue - img_id = line[0] - filename = f"{img_id}.jpg" - if filename != cp_filename: - data_infos.append(dict(filename=filename)) - cp_filename = filename - return data_infos - - -def get_metas_from_txt_style_ann_file(ann_file): - with open(ann_file) as f: - lines = f.readlines() - i = 0 - data_infos = [] - while i < len(lines): - filename = lines[i].rstrip() - data_infos.append(dict(filename=filename)) - skip_lines = int(lines[i + 2]) + 3 - i += skip_lines - return data_infos - - -def get_image_metas(data_info, img_prefix): - file_client = mmcv.FileClient(backend="disk") - filename = data_info.get("filename", None) - if filename is not None: - if img_prefix is not None: - filename = osp.join(img_prefix, filename) - img_bytes = file_client.get(filename) - img = mmcv.imfrombytes(img_bytes, flag="color") - meta = dict(filename=filename, ori_shape=img.shape) - else: - raise NotImplementedError("Missing `filename` in data_info") - return meta - - -def main(): - args = parse_args() - assert args.out.endswith("pkl"), "The output file name must be pkl suffix" - - # load config files - cfg = Config.fromfile(args.config) - ann_file = cfg.data.test.ann_file - img_prefix = cfg.data.test.img_prefix - - print(f"{'-' * 5} Start Processing {'-' * 5}") - if ann_file.endswith("csv"): - data_infos = get_metas_from_csv_style_ann_file(ann_file) - elif ann_file.endswith("txt"): - data_infos = get_metas_from_txt_style_ann_file(ann_file) - else: - shuffix = ann_file.split(".")[-1] - raise NotImplementedError(f"File name must be csv or txt suffix but get {shuffix}") - - print(f"Successfully load annotation file from {ann_file}") - print(f"Processing {len(data_infos)} images...") - pool = Pool(args.nproc) - # get image metas with multiple processes - image_metas = pool.starmap( - get_image_metas, - zip(data_infos, [img_prefix for _ in range(len(data_infos))]), - ) - pool.close() - - # save image metas - root_path = cfg.data.test.ann_file.rsplit("/", 1)[0] - save_path = osp.join(root_path, args.out) - mmcv.dump(image_metas, save_path) - print(f"Image meta file save to: {save_path}") - - -if __name__ == "__main__": - main() diff --git a/tools/misc/print_config.py b/tools/misc/print_config.py deleted file mode 100644 index 1722488e..00000000 --- a/tools/misc/print_config.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import warnings - -from mmcv import Config, DictAction -from mmdet.utils import replace_cfg_vals, update_data_root - - -def parse_args(): - parser = argparse.ArgumentParser(description="Print the whole config") - parser.add_argument("config", help="config file path") - parser.add_argument( - "--options", - nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file (deprecate), " - "change to --cfg-options instead.", - ) - parser.add_argument( - "--cfg-options", - nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file. If the value to " - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - "Note that the quotation marks are necessary and that no white space " - "is allowed.", - ) - args = parser.parse_args() - - if args.options and args.cfg_options: - raise ValueError( - "--options and --cfg-options cannot be both specified, --options is deprecated in favor of --cfg-options" - ) - if args.options: - warnings.warn("--options is deprecated in favor of --cfg-options") - args.cfg_options = args.options - - return args - - -def main(): - args = parse_args() - - cfg = Config.fromfile(args.config) - - # replace the ${key} with the value of cfg.key - cfg = replace_cfg_vals(cfg) - - # update data root according to MMDET_DATASETS - update_data_root(cfg) - - if args.cfg_options is not None: - cfg.merge_from_dict(args.cfg_options) - print(f"Config:\n{cfg.pretty_text}") - - -if __name__ == "__main__": - main() diff --git a/tools/misc/split_coco.py b/tools/misc/split_coco.py deleted file mode 100644 index 094e1aa1..00000000 --- a/tools/misc/split_coco.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import os.path as osp - -import mmcv -import numpy as np - -prog_description = """K-Fold coco split. - -To split coco data for semi-supervised object detection: - python tools/misc/split_coco.py -""" - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--data-root", - type=str, - help="The data root of coco dataset.", - default="./data/coco/", - ) - parser.add_argument( - "--out-dir", - type=str, - help="The output directory of coco semi-supervised annotations.", - default="./data/coco_semi_annos/", - ) - parser.add_argument( - "--labeled-percent", - type=float, - nargs="+", - help="The percentage of labeled data in the training set.", - default=[1, 2, 5, 10], - ) - parser.add_argument( - "--fold", - type=int, - help="K-fold cross validation for semi-supervised object detection.", - default=5, - ) - args = parser.parse_args() - return args - - -def split_coco(data_root, out_dir, percent, fold): - """Split COCO data for Semi-supervised object detection. - - Args: - data_root (str): The data root of coco dataset. - out_dir (str): The output directory of coco semi-supervised - annotations. - percent (float): The percentage of labeled data in the training set. - fold (int): The fold of dataset and set as random seed for data split. - """ - - def save_anns(name, images, annotations): - sub_anns = dict() - sub_anns["images"] = images - sub_anns["annotations"] = annotations - sub_anns["licenses"] = anns["licenses"] - sub_anns["categories"] = anns["categories"] - sub_anns["info"] = anns["info"] - - mmcv.mkdir_or_exist(out_dir) - mmcv.dump(sub_anns, f"{out_dir}/{name}.json") - - # set random seed with the fold - np.random.seed(fold) - ann_file = osp.join(data_root, "annotations/instances_train2017.json") - anns = mmcv.load(ann_file) - - image_list = anns["images"] - labeled_total = int(percent / 100.0 * len(image_list)) - labeled_inds = set(np.random.choice(range(len(image_list)), size=labeled_total)) - labeled_ids, labeled_images, unlabeled_images = [], [], [] - - for i in range(len(image_list)): - if i in labeled_inds: - labeled_images.append(image_list[i]) - labeled_ids.append(image_list[i]["id"]) - else: - unlabeled_images.append(image_list[i]) - - # get all annotations of labeled images - labeled_ids = set(labeled_ids) - labeled_annotations, unlabeled_annotations = [], [] - - for ann in anns["annotations"]: - if ann["image_id"] in labeled_ids: - labeled_annotations.append(ann) - else: - unlabeled_annotations.append(ann) - - # save labeled and unlabeled - labeled_name = f"instances_train2017.{fold}@{percent}" - unlabeled_name = f"instances_train2017.{fold}@{percent}-unlabeled" - - save_anns(labeled_name, labeled_images, labeled_annotations) - save_anns(unlabeled_name, unlabeled_images, unlabeled_annotations) - - -def multi_wrapper(args): - return split_coco(*args) - - -if __name__ == "__main__": - args = parse_args() - arguments_list = [ - (args.data_root, args.out_dir, p, f) for f in range(1, args.fold + 1) for p in args.labeled_percent - ] - mmcv.track_parallel_progress(multi_wrapper, arguments_list, args.fold) diff --git a/tools/model_converters/detectron2pytorch.py b/tools/model_converters/detectron2pytorch.py deleted file mode 100644 index 6fbd26ab..00000000 --- a/tools/model_converters/detectron2pytorch.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -from collections import OrderedDict - -import mmcv -import torch - -arch_settings = {50: (3, 4, 6, 3), 101: (3, 4, 23, 3)} - - -def convert_bn(blobs, state_dict, caffe_name, torch_name, converted_names): - # detectron replace bn with affine channel layer - state_dict[torch_name + ".bias"] = torch.from_numpy(blobs[caffe_name + "_b"]) - state_dict[torch_name + ".weight"] = torch.from_numpy(blobs[caffe_name + "_s"]) - bn_size = state_dict[torch_name + ".weight"].size() - state_dict[torch_name + ".running_mean"] = torch.zeros(bn_size) - state_dict[torch_name + ".running_var"] = torch.ones(bn_size) - converted_names.add(caffe_name + "_b") - converted_names.add(caffe_name + "_s") - - -def convert_conv_fc(blobs, state_dict, caffe_name, torch_name, converted_names): - state_dict[torch_name + ".weight"] = torch.from_numpy(blobs[caffe_name + "_w"]) - converted_names.add(caffe_name + "_w") - if caffe_name + "_b" in blobs: - state_dict[torch_name + ".bias"] = torch.from_numpy(blobs[caffe_name + "_b"]) - converted_names.add(caffe_name + "_b") - - -def convert(src, dst, depth): - """Convert keys in detectron pretrained ResNet models to pytorch style.""" - # load arch_settings - if depth not in arch_settings: - raise ValueError("Only support ResNet-50 and ResNet-101 currently") - block_nums = arch_settings[depth] - # load caffe model - caffe_model = mmcv.load(src, encoding="latin1") - blobs = caffe_model["blobs"] if "blobs" in caffe_model else caffe_model - # convert to pytorch style - state_dict = OrderedDict() - converted_names = set() - convert_conv_fc(blobs, state_dict, "conv1", "conv1", converted_names) - convert_bn(blobs, state_dict, "res_conv1_bn", "bn1", converted_names) - for i in range(1, len(block_nums) + 1): - for j in range(block_nums[i - 1]): - if j == 0: - convert_conv_fc( - blobs, - state_dict, - f"res{i + 1}_{j}_branch1", - f"layer{i}.{j}.downsample.0", - converted_names, - ) - convert_bn( - blobs, - state_dict, - f"res{i + 1}_{j}_branch1_bn", - f"layer{i}.{j}.downsample.1", - converted_names, - ) - for k, letter in enumerate(["a", "b", "c"]): - convert_conv_fc( - blobs, - state_dict, - f"res{i + 1}_{j}_branch2{letter}", - f"layer{i}.{j}.conv{k + 1}", - converted_names, - ) - convert_bn( - blobs, - state_dict, - f"res{i + 1}_{j}_branch2{letter}_bn", - f"layer{i}.{j}.bn{k + 1}", - converted_names, - ) - # check if all layers are converted - for key in blobs: - if key not in converted_names: - print(f"Not Convert: {key}") - # save checkpoint - checkpoint = dict() - checkpoint["state_dict"] = state_dict - torch.save(checkpoint, dst) - - -def main(): - parser = argparse.ArgumentParser(description="Convert model keys") - parser.add_argument("src", help="src detectron model path") - parser.add_argument("dst", help="save path") - parser.add_argument("depth", type=int, help="ResNet model depth") - args = parser.parse_args() - convert(args.src, args.dst, args.depth) - - -if __name__ == "__main__": - main() diff --git a/tools/model_converters/upgrade_model_version.py b/tools/model_converters/upgrade_model_version.py deleted file mode 100644 index e9bd0ffe..00000000 --- a/tools/model_converters/upgrade_model_version.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import re -import tempfile -from collections import OrderedDict - -import torch -from mmcv import Config - - -def is_head(key): - valid_head_list = [ - "bbox_head", - "mask_head", - "semantic_head", - "grid_head", - "mask_iou_head", - ] - - return any(key.startswith(h) for h in valid_head_list) - - -def parse_config(config_strings): - temp_file = tempfile.NamedTemporaryFile() - config_path = f"{temp_file.name}.py" - with open(config_path, "w") as f: - f.write(config_strings) - - config = Config.fromfile(config_path) - is_two_stage = True - is_ssd = False - is_retina = False - reg_cls_agnostic = False - if "rpn_head" not in config.model: - is_two_stage = False - # check whether it is SSD - if config.model.bbox_head.type == "SSDHead": - is_ssd = True - elif config.model.bbox_head.type == "RetinaHead": - is_retina = True - elif isinstance(config.model["bbox_head"], list): - reg_cls_agnostic = True - elif "reg_class_agnostic" in config.model.bbox_head: - reg_cls_agnostic = config.model.bbox_head.reg_class_agnostic - temp_file.close() - return is_two_stage, is_ssd, is_retina, reg_cls_agnostic - - -def reorder_cls_channel(val, num_classes=81): - # bias - if val.dim() == 1: - new_val = torch.cat((val[1:], val[:1]), dim=0) - # weight - else: - out_channels, in_channels = val.shape[:2] - # conv_cls for softmax output - if out_channels != num_classes and out_channels % num_classes == 0: - new_val = val.reshape(-1, num_classes, in_channels, *val.shape[2:]) - new_val = torch.cat((new_val[:, 1:], new_val[:, :1]), dim=1) - new_val = new_val.reshape(val.size()) - # fc_cls - elif out_channels == num_classes: - new_val = torch.cat((val[1:], val[:1]), dim=0) - # agnostic | retina_cls | rpn_cls - else: - new_val = val - - return new_val - - -def truncate_cls_channel(val, num_classes=81): - # bias - if val.dim() == 1: - if val.size(0) % num_classes == 0: - new_val = val[: num_classes - 1] - else: - new_val = val - # weight - else: - out_channels, in_channels = val.shape[:2] - # conv_logits - if out_channels % num_classes == 0: - new_val = val.reshape(num_classes, in_channels, *val.shape[2:])[1:] - new_val = new_val.reshape(-1, *val.shape[1:]) - # agnostic - else: - new_val = val - - return new_val - - -def truncate_reg_channel(val, num_classes=81): - # bias - if val.dim() == 1: - # fc_reg | rpn_reg - if val.size(0) % num_classes == 0: - new_val = val.reshape(num_classes, -1)[: num_classes - 1] - new_val = new_val.reshape(-1) - # agnostic - else: - new_val = val - # weight - else: - out_channels, in_channels = val.shape[:2] - # fc_reg | rpn_reg - if out_channels % num_classes == 0: - new_val = val.reshape(num_classes, -1, in_channels, *val.shape[2:])[1:] - new_val = new_val.reshape(-1, *val.shape[1:]) - # agnostic - else: - new_val = val - - return new_val - - -def convert(in_file, out_file, num_classes): - """Convert keys in checkpoints. - - There can be some breaking changes during the development of mmdetection, - and this tool is used for upgrading checkpoints trained with old versions - to the latest one. - """ - checkpoint = torch.load(in_file) - in_state_dict = checkpoint.pop("state_dict") - out_state_dict = OrderedDict() - meta_info = checkpoint["meta"] - is_two_stage, is_ssd, is_retina, reg_cls_agnostic = parse_config("#" + meta_info["config"]) - if meta_info["mmdet_version"] <= "0.5.3" and is_retina: - upgrade_retina = True - else: - upgrade_retina = False - - # MMDetection v2.5.0 unifies the class order in RPN - # if the model is trained in version=2.5.0 - if meta_info["mmdet_version"] < "2.5.0": - upgrade_rpn = True - else: - upgrade_rpn = False - - for key, val in in_state_dict.items(): - new_key = key - new_val = val - if is_two_stage and is_head(key): - new_key = f"roi_head.{key}" - - # classification - if upgrade_rpn: - m = re.search( - r"(conv_cls|retina_cls|rpn_cls|fc_cls|fcos_cls|" - r"fovea_cls).(weight|bias)", - new_key, - ) - else: - m = re.search( - r"(conv_cls|retina_cls|fc_cls|fcos_cls|" - r"fovea_cls).(weight|bias)", - new_key, - ) - if m is not None: - print(f"reorder cls channels of {new_key}") - new_val = reorder_cls_channel(val, num_classes) - - # regression - if upgrade_rpn: - m = re.search(r"(fc_reg).(weight|bias)", new_key) - else: - m = re.search(r"(fc_reg|rpn_reg).(weight|bias)", new_key) - if m is not None and not reg_cls_agnostic: - print(f"truncate regression channels of {new_key}") - new_val = truncate_reg_channel(val, num_classes) - - # mask head - m = re.search(r"(conv_logits).(weight|bias)", new_key) - if m is not None: - print(f"truncate mask prediction channels of {new_key}") - new_val = truncate_cls_channel(val, num_classes) - - m = re.search(r"(cls_convs|reg_convs).\d.(weight|bias)", key) - # Legacy issues in RetinaNet since V1.x - # Use ConvModule instead of nn.Conv2d in RetinaNet - # cls_convs.0.weight -> cls_convs.0.conv.weight - if m is not None and upgrade_retina: - param = m.groups()[1] - new_key = key.replace(param, f"conv.{param}") - out_state_dict[new_key] = val - print(f"rename the name of {key} to {new_key}") - continue - - m = re.search(r"(cls_convs).\d.(weight|bias)", key) - if m is not None and is_ssd: - print(f"reorder cls channels of {new_key}") - new_val = reorder_cls_channel(val, num_classes) - - out_state_dict[new_key] = new_val - checkpoint["state_dict"] = out_state_dict - torch.save(checkpoint, out_file) - - -def main(): - parser = argparse.ArgumentParser(description="Upgrade model version") - parser.add_argument("in_file", help="input checkpoint file") - parser.add_argument("out_file", help="output checkpoint file") - parser.add_argument( - "--num-classes", - type=int, - default=81, - help="number of classes of the original model", - ) - args = parser.parse_args() - convert(args.in_file, args.out_file, args.num_classes) - - -if __name__ == "__main__": - main() diff --git a/tools/model_converters/upgrade_ssd_version.py b/tools/model_converters/upgrade_ssd_version.py deleted file mode 100644 index 1bde6847..00000000 --- a/tools/model_converters/upgrade_ssd_version.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import tempfile -from collections import OrderedDict - -import torch -from mmcv import Config - - -def parse_config(config_strings): - temp_file = tempfile.NamedTemporaryFile() - config_path = f"{temp_file.name}.py" - with open(config_path, "w") as f: - f.write(config_strings) - - config = Config.fromfile(config_path) - # check whether it is SSD - if config.model.bbox_head.type != "SSDHead": - raise AssertionError("This is not a SSD model.") - - -def convert(in_file, out_file): - checkpoint = torch.load(in_file) - in_state_dict = checkpoint.pop("state_dict") - out_state_dict = OrderedDict() - meta_info = checkpoint["meta"] - parse_config("#" + meta_info["config"]) - for key, value in in_state_dict.items(): - if "extra" in key: - layer_idx = int(key.split(".")[2]) - new_key = f"neck.extra_layers.{layer_idx // 2}.{layer_idx % 2}.conv." + key.split(".")[-1] - elif "l2_norm" in key: - new_key = "neck.l2_norm.weight" - elif "bbox_head" in key: - new_key = key[:21] + ".0" + key[21:] - else: - new_key = key - out_state_dict[new_key] = value - checkpoint["state_dict"] = out_state_dict - - if torch.__version__ >= "1.6": - torch.save(checkpoint, out_file, _use_new_zipfile_serialization=False) - else: - torch.save(checkpoint, out_file) - - -def main(): - parser = argparse.ArgumentParser(description="Upgrade SSD version") - parser.add_argument("in_file", help="input checkpoint file") - parser.add_argument("out_file", help="output checkpoint file") - - args = parser.parse_args() - convert(args.in_file, args.out_file) - - -if __name__ == "__main__": - main() diff --git a/tools/test.py b/tools/test.py index c6d86edf..07645928 100644 --- a/tools/test.py +++ b/tools/test.py @@ -1,113 +1,84 @@ -# Copyright (c) OpenMMLab. All rights reserved. +"""Test/evaluate a detector with visdet-native runner config. + +This entrypoint intentionally avoids mmcv/mmdet/mmengine package dependencies. +""" + import argparse +import json import os -import os.path as osp -import time import warnings +from typing import Any + +import yaml + +from visdet.engine import DefaultScope +from visdet.engine.config import Config +from visdet.engine.runner import Runner + + +def _parse_cfg_options(values: list[str] | None) -> dict[str, Any]: + parsed: dict[str, Any] = {} + if not values: + return parsed + + for item in values: + if "=" not in item: + raise ValueError(f"Invalid cfg option '{item}'. Expected key=value format.") + key, value = item.split("=", 1) + try: + parsed[key] = yaml.safe_load(value) + except Exception: + parsed[key] = value + return parsed + + +def _apply_eval_metrics(cfg: Config, metrics: list[str]) -> None: + metric_value: str | list[str] = metrics if len(metrics) > 1 else metrics[0] + evaluator = cfg.get("test_evaluator", None) + if evaluator is None: + evaluator = cfg.get("val_evaluator", None) + if evaluator is not None: + cfg.test_evaluator = evaluator + + if evaluator is None: + raise ValueError("No test_evaluator/val_evaluator found in config; cannot apply --eval metrics.") + + if isinstance(evaluator, dict): + evaluator["metric"] = metric_value + elif isinstance(evaluator, list): + for item in evaluator: + if isinstance(item, dict): + item["metric"] = metric_value + else: + raise TypeError(f"Unsupported evaluator type: {type(evaluator)}") -import mmcv -import torch -from mmcv import Config, DictAction -from mmcv.cnn import fuse_conv_bn -from mmcv.runner import get_dist_info, init_dist, load_checkpoint, wrap_fp16_model -from mmdet.apis import multi_gpu_test, single_gpu_test -from mmdet.datasets import build_dataloader, build_dataset, replace_ImageToTensor -from mmdet.models import build_detector -from mmdet.utils import ( - build_ddp, - build_dp, - compat_cfg, - get_device, - replace_cfg_vals, - rfnext_init_model, - setup_multi_processes, - update_data_root, -) - - -def parse_args(): - parser = argparse.ArgumentParser(description="MMDet test (and eval) a model") + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Test (and eval) a model (visdet-native)") parser.add_argument("config", help="test config file path") - parser.add_argument("checkpoint", help="checkpoint file") - parser.add_argument( - "--work-dir", - help="the directory to save the file containing evaluation metrics", - ) - parser.add_argument("--out", help="output result file in pickle format") - parser.add_argument( - "--fuse-conv-bn", - action="store_true", - help="Whether to fuse conv and bn, this will slightly increasethe inference speed", - ) - parser.add_argument( - "--gpu-ids", - type=int, - nargs="+", - help="(Deprecated, please use --gpu-id) ids of gpus to use (only applicable to non-distributed training)", - ) - parser.add_argument( - "--gpu-id", - type=int, - default=0, - help="id of gpu to use (only applicable to non-distributed testing)", - ) + parser.add_argument("checkpoint", help="checkpoint file path") + parser.add_argument("--work-dir", help="directory to save evaluation metrics") + parser.add_argument("--out", help="output JSON file for metrics") + parser.add_argument("--eval", type=str, nargs="+", help='evaluation metrics, e.g., "bbox segm"') parser.add_argument( "--format-only", action="store_true", - help="Format the output results without perform evaluation. It is" - "useful when you want to format the result to a specific format and " - "submit it to the test server", - ) - parser.add_argument( - "--eval", - type=str, - nargs="+", - help='evaluation metrics, which depends on the dataset, e.g., "bbox",' - ' "segm", "proposal" for COCO, and "mAP", "recall" for PASCAL VOC', - ) - parser.add_argument("--show", action="store_true", help="show results") - parser.add_argument("--show-dir", help="directory where painted images will be saved") - parser.add_argument( - "--show-score-thr", - type=float, - default=0.3, - help="score threshold (default: 0.3)", - ) - parser.add_argument( - "--gpu-collect", - action="store_true", - help="whether to use gpu to collect results.", - ) - parser.add_argument( - "--tmpdir", - help="tmp directory used for collecting results from multiple " - "workers, available when gpu-collect is not specified", + help="format-only mode from legacy mmdet toolchain (not supported in visdet-native runner)", ) parser.add_argument( "--cfg-options", nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file. If the value to " - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - "Note that the quotation marks are necessary and that no white space " - "is allowed.", + help="override config in key=value format (e.g. test_dataloader.batch_size=1)", ) parser.add_argument( "--options", nargs="+", - action=DictAction, - help="custom options for evaluation, the key-value pair in xxx=yyy " - "format will be kwargs for dataset.evaluate() function (deprecate), " - "change to --eval-options instead.", + help="deprecated alias for --eval-options", ) parser.add_argument( "--eval-options", nargs="+", - action=DictAction, - help="custom options for evaluation, the key-value pair in xxx=yyy " - "format will be kwargs for dataset.evaluate() function", + help="custom evaluation options in key=value format (merged into cfg)", ) parser.add_argument( "--launcher", @@ -116,190 +87,74 @@ def parse_args(): help="job launcher", ) parser.add_argument("--local_rank", type=int, default=0) + + # Deprecated/ignored options retained for CLI compatibility. + parser.add_argument("--fuse-conv-bn", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--gpu-ids", type=int, nargs="+", help=argparse.SUPPRESS) + parser.add_argument("--gpu-id", type=int, default=0, help=argparse.SUPPRESS) + parser.add_argument("--show", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--show-dir", help=argparse.SUPPRESS) + parser.add_argument("--show-score-thr", type=float, default=0.3, help=argparse.SUPPRESS) + parser.add_argument("--gpu-collect", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--tmpdir", help=argparse.SUPPRESS) + args = parser.parse_args() + if "LOCAL_RANK" not in os.environ: os.environ["LOCAL_RANK"] = str(args.local_rank) if args.options and args.eval_options: - raise ValueError( - "--options and --eval-options cannot be both specified, --options is deprecated in favor of --eval-options" - ) + raise ValueError("--options and --eval-options cannot be both specified") if args.options: - warnings.warn("--options is deprecated in favor of --eval-options") + warnings.warn("--options is deprecated in favor of --eval-options", stacklevel=2) args.eval_options = args.options + return args -def main(): +def main() -> None: args = parse_args() - assert args.out or args.eval or args.format_only or args.show or args.show_dir, ( - "Please specify at least one operation (save/eval/format/show the " - 'results / save the results) with the argument "--out", "--eval"' - ', "--format-only", "--show" or "--show-dir"' - ) - - if args.eval and args.format_only: - raise ValueError("--eval and --format_only cannot be both specified") + if args.format_only: + raise ValueError("--format-only is not supported by visdet-native tools/test.py") - if args.out is not None and not args.out.endswith((".pkl", ".pickle")): - raise ValueError("The output file must be a pkl file.") + if args.fuse_conv_bn: + warnings.warn("--fuse-conv-bn is ignored by visdet-native tools/test.py", stacklevel=2) cfg = Config.fromfile(args.config) - # replace the ${key} with the value of cfg.key - cfg = replace_cfg_vals(cfg) - - # update data root according to MMDET_DATASETS - update_data_root(cfg) - - if args.cfg_options is not None: - cfg.merge_from_dict(args.cfg_options) - - cfg = compat_cfg(cfg) - - # set multi-process settings - setup_multi_processes(cfg) - - # set cudnn_benchmark - if cfg.get("cudnn_benchmark", False): - torch.backends.cudnn.benchmark = True - - if "pretrained" in cfg.model: - cfg.model.pretrained = None - elif "init_cfg" in cfg.model.backbone: - cfg.model.backbone.init_cfg = None - - if cfg.model.get("neck"): - if isinstance(cfg.model.neck, list): - for neck_cfg in cfg.model.neck: - if neck_cfg.get("rfp_backbone"): - if neck_cfg.rfp_backbone.get("pretrained"): - neck_cfg.rfp_backbone.pretrained = None - elif cfg.model.neck.get("rfp_backbone"): - if cfg.model.neck.rfp_backbone.get("pretrained"): - cfg.model.neck.rfp_backbone.pretrained = None - - if args.gpu_ids is not None: - cfg.gpu_ids = args.gpu_ids[0:1] - warnings.warn( - "`--gpu-ids` is deprecated, please use `--gpu-id`. " - "Because we only support single GPU mode in " - "non-distributed testing. Use the first GPU " - "in `gpu_ids` now." - ) - else: - cfg.gpu_ids = [args.gpu_id] - cfg.device = get_device() - # init distributed env first, since logger depends on the dist info. - if args.launcher == "none": - distributed = False - else: - distributed = True - init_dist(args.launcher, **cfg.dist_params) - - test_dataloader_default_args = dict(samples_per_gpu=1, workers_per_gpu=2, dist=distributed, shuffle=False) - - # in case the test dataset is concatenated - if isinstance(cfg.data.test, dict): - cfg.data.test.test_mode = True - if cfg.data.test_dataloader.get("samples_per_gpu", 1) > 1: - # Replace 'ImageToTensor' to 'DefaultFormatBundle' - cfg.data.test.pipeline = replace_ImageToTensor(cfg.data.test.pipeline) - elif isinstance(cfg.data.test, list): - for ds_cfg in cfg.data.test: - ds_cfg.test_mode = True - if cfg.data.test_dataloader.get("samples_per_gpu", 1) > 1: - for ds_cfg in cfg.data.test: - ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) - - test_loader_cfg = { - **test_dataloader_default_args, - **cfg.data.get("test_dataloader", {}), - } - - rank, _ = get_dist_info() - # allows not to create - if args.work_dir is not None and rank == 0: - mmcv.mkdir_or_exist(osp.abspath(args.work_dir)) - timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime()) - json_file = osp.join(args.work_dir, f"eval_{timestamp}.json") - - # build the dataloader - dataset = build_dataset(cfg.data.test) - data_loader = build_dataloader(dataset, **test_loader_cfg) - - # build the model and load checkpoint - cfg.model.train_cfg = None - model = build_detector(cfg.model, test_cfg=cfg.get("test_cfg")) - # init rfnext if 'RFSearchHook' is defined in cfg - rfnext_init_model(model, cfg=cfg) - fp16_cfg = cfg.get("fp16", None) - if fp16_cfg is None and cfg.get("device", None) == "npu": - fp16_cfg = dict(loss_scale="dynamic") - if fp16_cfg is not None: - wrap_fp16_model(model) - checkpoint = load_checkpoint(model, args.checkpoint, map_location="cpu") - if args.fuse_conv_bn: - model = fuse_conv_bn(model) - # old versions did not save class info in checkpoints, this walkaround is - # for backward compatibility - if "CLASSES" in checkpoint.get("meta", {}): - model.CLASSES = checkpoint["meta"]["CLASSES"] - else: - model.CLASSES = dataset.CLASSES + if args.cfg_options: + cfg.merge_from_dict(_parse_cfg_options(args.cfg_options)) - if not distributed: - model = build_dp(model, cfg.device, device_ids=cfg.gpu_ids) - outputs = single_gpu_test(model, data_loader, args.show, args.show_dir, args.show_score_thr) - else: - model = build_ddp( - model, - cfg.device, - device_ids=[int(os.environ["LOCAL_RANK"])], - broadcast_buffers=False, - ) - - # In multi_gpu_test, if tmpdir is None, some tesnors - # will init on cuda by default, and no device choice supported. - # Init a tmpdir to avoid error on npu here. - if cfg.device == "npu" and args.tmpdir is None: - args.tmpdir = "./npu_tmpdir" - - outputs = multi_gpu_test( - model, - data_loader, - args.tmpdir, - args.gpu_collect or cfg.evaluation.get("gpu_collect", False), - ) - - rank, _ = get_dist_info() - if rank == 0: - if args.out: - print(f"\nwriting results to {args.out}") - mmcv.dump(outputs, args.out) - kwargs = {} if args.eval_options is None else args.eval_options - if args.format_only: - dataset.format_results(outputs, **kwargs) - if args.eval: - eval_kwargs = cfg.get("evaluation", {}).copy() - # hard-code way to remove EvalHook args - for key in [ - "interval", - "tmpdir", - "start", - "gpu_collect", - "save_best", - "rule", - "dynamic_intervals", - ]: - eval_kwargs.pop(key, None) - eval_kwargs.update(dict(metric=args.eval, **kwargs)) - metric = dataset.evaluate(outputs, **eval_kwargs) - print(metric) - metric_dict = dict(config=args.config, metric=metric) - if args.work_dir is not None and rank == 0: - mmcv.dump(metric_dict, json_file) + if args.eval_options: + cfg.merge_from_dict(_parse_cfg_options(args.eval_options)) + + if args.work_dir: + cfg.work_dir = args.work_dir + + cfg.launcher = args.launcher + cfg.load_from = args.checkpoint + cfg.resume = False + + if args.eval: + _apply_eval_metrics(cfg, args.eval) + + DefaultScope.get_instance("visdet", scope_name="visdet") + runner = Runner.from_cfg(cfg) + metrics = runner.test() + + if metrics: + print(metrics) + + output_path = args.out + if output_path is None and args.work_dir: + output_path = os.path.join(args.work_dir, "eval_metrics.json") + + if output_path: + os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + json.dump({"config": args.config, "checkpoint": args.checkpoint, "metrics": metrics}, f, indent=2) + print(f"Saved metrics to {output_path}") if __name__ == "__main__": diff --git a/tools/train.py b/tools/train.py index 413c0a73..0a68d6a7 100644 --- a/tools/train.py +++ b/tools/train.py @@ -1,94 +1,78 @@ -# Copyright (c) OpenMMLab. All rights reserved. +"""Train a detector with visdet-native runner config. + +This entrypoint intentionally avoids mmcv/mmdet/mmengine package dependencies. +""" + import argparse -import copy import os -import os.path as osp -import time import warnings +from typing import Any + +import yaml + +from visdet.engine import DefaultScope +from visdet.engine.config import Config +from visdet.engine.runner import Runner + -import mmcv -import torch -import torch.distributed as dist -from mmcv import Config, DictAction -from mmcv.runner import get_dist_info, init_dist -from mmcv.utils import get_git_hash -from mmdet import __version__ -from mmdet.apis import init_random_seed, set_random_seed, train_detector -from mmdet.datasets import build_dataset -from mmdet.models import build_detector -from mmdet.utils import ( - collect_env, - get_device, - get_root_logger, - replace_cfg_vals, - rfnext_init_model, - setup_multi_processes, - update_data_root, -) - - -def parse_args(): - parser = argparse.ArgumentParser(description="Train a detector") +def _parse_cfg_options(values: list[str] | None) -> dict[str, Any]: + """Parse key=value CLI overrides into a dict for Config.merge_from_dict().""" + parsed: dict[str, Any] = {} + if not values: + return parsed + + for item in values: + if "=" not in item: + raise ValueError(f"Invalid cfg option '{item}'. Expected key=value format.") + key, value = item.split("=", 1) + try: + parsed[key] = yaml.safe_load(value) + except Exception: + parsed[key] = value + return parsed + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Train a detector (visdet-native)") parser.add_argument("config", help="train config file path") parser.add_argument("--work-dir", help="the dir to save logs and models") - parser.add_argument("--resume-from", help="the checkpoint file to resume from") + parser.add_argument("--resume-from", help="checkpoint file to resume from") + parser.add_argument("--load-from", help="checkpoint file to load model weights from") parser.add_argument( "--auto-resume", action="store_true", - help="resume from the latest checkpoint automatically", + help="resume from latest checkpoint in work_dir", ) parser.add_argument( "--no-validate", action="store_true", - help="whether not to evaluate the checkpoint during training", - ) - group_gpus = parser.add_mutually_exclusive_group() - group_gpus.add_argument( - "--gpus", - type=int, - help="(Deprecated, please use --gpu-id) number of gpus to use (only applicable to non-distributed training)", - ) - group_gpus.add_argument( - "--gpu-ids", - type=int, - nargs="+", - help="(Deprecated, please use --gpu-id) ids of gpus to use (only applicable to non-distributed training)", + help="disable validation loop during training", ) - group_gpus.add_argument( - "--gpu-id", - type=int, - default=0, - help="id of gpu to use (only applicable to non-distributed training)", + parser.add_argument( + "--fp16", + action="store_true", + help="enable AMP optimizer wrapper for fp16 training", ) parser.add_argument("--seed", type=int, default=None, help="random seed") parser.add_argument( "--diff-seed", action="store_true", - help="Whether or not set different seeds for different ranks", + help="offset seed by LOCAL_RANK for distributed launchers", ) parser.add_argument( "--deterministic", action="store_true", - help="whether to set deterministic options for CUDNN backend.", + help="set deterministic options for reproducibility", ) parser.add_argument( "--options", nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file (deprecate), " - "change to --cfg-options instead.", + help="deprecated alias for --cfg-options", ) parser.add_argument( "--cfg-options", nargs="+", - action=DictAction, - help="override some settings in the used config, the key-value pair " - "in xxx=yyy format will be merged into config file. If the value to " - 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' - 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - "Note that the quotation marks are necessary and that no white space " - "is allowed.", + help="override config in key=value format (e.g. optimizer.lr=0.001 data.train.batch_size=2)", ) parser.add_argument( "--launcher", @@ -97,154 +81,74 @@ def parse_args(): help="job launcher", ) parser.add_argument("--local_rank", type=int, default=0) - parser.add_argument("--auto-scale-lr", action="store_true", help="enable automatically scaling LR.") + parser.add_argument("--auto-scale-lr", action="store_true", help="enable auto_scale_lr in config if present") + # Deprecated/ignored GPU flags retained for CLI compatibility. + parser.add_argument("--gpus", type=int, help=argparse.SUPPRESS) + parser.add_argument("--gpu-ids", type=int, nargs="+", help=argparse.SUPPRESS) + parser.add_argument("--gpu-id", type=int, default=0, help=argparse.SUPPRESS) args = parser.parse_args() - if "LOCAL_RANK" not in os.environ: - os.environ["LOCAL_RANK"] = str(args.local_rank) if args.options and args.cfg_options: - raise ValueError( - "--options and --cfg-options cannot be both specified, --options is deprecated in favor of --cfg-options" - ) + raise ValueError("--options and --cfg-options cannot be both specified") if args.options: - warnings.warn("--options is deprecated in favor of --cfg-options") + warnings.warn("--options is deprecated in favor of --cfg-options", stacklevel=2) args.cfg_options = args.options + if "LOCAL_RANK" not in os.environ: + os.environ["LOCAL_RANK"] = str(args.local_rank) return args -def main(): +def main() -> None: args = parse_args() cfg = Config.fromfile(args.config) - # replace the ${key} with the value of cfg.key - cfg = replace_cfg_vals(cfg) - - # update data root according to MMDET_DATASETS - update_data_root(cfg) - - if args.cfg_options is not None: - cfg.merge_from_dict(args.cfg_options) - - if args.auto_scale_lr: - if "auto_scale_lr" in cfg and "enable" in cfg.auto_scale_lr and "base_batch_size" in cfg.auto_scale_lr: - cfg.auto_scale_lr.enable = True - else: - warnings.warn( - 'Can not find "auto_scale_lr" or ' - '"auto_scale_lr.enable" or ' - '"auto_scale_lr.base_batch_size" in your' - " configuration file. Please update all the " - "configuration files to mmdet >= 2.24.1." - ) - - # set multi-process settings - setup_multi_processes(cfg) - - # set cudnn_benchmark - if cfg.get("cudnn_benchmark", False): - torch.backends.cudnn.benchmark = True - - # work_dir is determined in this priority: CLI > segment in file > filename - if args.work_dir is not None: - # update configs according to CLI args if args.work_dir is not None + if args.cfg_options: + cfg.merge_from_dict(_parse_cfg_options(args.cfg_options)) + + if args.work_dir: cfg.work_dir = args.work_dir - elif cfg.get("work_dir", None) is None: - # use config filename as default work_dir if cfg.work_dir is None - cfg.work_dir = osp.join("./work_dirs", osp.splitext(osp.basename(args.config))[0]) - - if args.resume_from is not None: - cfg.resume_from = args.resume_from - cfg.auto_resume = args.auto_resume - if args.gpus is not None: - cfg.gpu_ids = range(1) - warnings.warn( - "`--gpus` is deprecated because we only support " - "single GPU mode in non-distributed training. " - "Use `gpus=1` now." - ) - if args.gpu_ids is not None: - cfg.gpu_ids = args.gpu_ids[0:1] - warnings.warn( - "`--gpu-ids` is deprecated, please use `--gpu-id`. " - "Because we only support single GPU mode in " - "non-distributed training. Use the first GPU " - "in `gpu_ids` now." - ) - if args.gpus is None and args.gpu_ids is None: - cfg.gpu_ids = [args.gpu_id] - - # init distributed env first, since logger depends on the dist info. - if args.launcher == "none": - distributed = False - else: - distributed = True - init_dist(args.launcher, **cfg.dist_params) - # re-set gpu_ids with distributed training mode - _, world_size = get_dist_info() - cfg.gpu_ids = range(world_size) - - # create work_dir - mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir)) - # dump config - cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config))) - # init the logger before other steps - timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime()) - log_file = osp.join(cfg.work_dir, f"{timestamp}.log") - logger = get_root_logger(log_file=log_file, log_level=cfg.log_level) - - # init the meta dict to record some important information such as - # environment info and seed, which will be logged - meta = dict() - # log env info - env_info_dict = collect_env() - env_info = "\n".join([(f"{k}: {v}") for k, v in env_info_dict.items()]) - dash_line = "-" * 60 + "\n" - logger.info("Environment info:\n" + dash_line + env_info + "\n" + dash_line) - meta["env_info"] = env_info - meta["config"] = cfg.pretty_text - # log some basic info - logger.info(f"Distributed training: {distributed}") - logger.info(f"Config:\n{cfg.pretty_text}") - - cfg.device = get_device() - # set random seeds - seed = init_random_seed(args.seed, device=cfg.device) - seed = seed + dist.get_rank() if args.diff_seed else seed - logger.info(f"Set random seed to {seed}, deterministic: {args.deterministic}") - set_random_seed(seed, deterministic=args.deterministic) - cfg.seed = seed - meta["seed"] = seed - meta["exp_name"] = osp.basename(args.config) - - model = build_detector(cfg.model, train_cfg=cfg.get("train_cfg"), test_cfg=cfg.get("test_cfg")) - model.init_weights() - - # init rfnext if 'RFSearchHook' is defined in cfg - rfnext_init_model(model, cfg=cfg) - - datasets = [build_dataset(cfg.data.train)] - if len(cfg.workflow) == 2: - assert "val" in [mode for (mode, _) in cfg.workflow] - val_dataset = copy.deepcopy(cfg.data.val) - val_dataset.pipeline = cfg.data.train.get("pipeline", cfg.data.train.dataset.get("pipeline")) - datasets.append(build_dataset(val_dataset)) - if cfg.checkpoint_config is not None: - # save mmdet version, config file content and class names in - # checkpoints as meta data - cfg.checkpoint_config.meta = dict(mmdet_version=__version__ + get_git_hash()[:7], CLASSES=datasets[0].CLASSES) - # add an attribute for visualization convenience - model.CLASSES = datasets[0].CLASSES - train_detector( - model, - datasets, - cfg, - distributed=distributed, - validate=(not args.no_validate), - timestamp=timestamp, - meta=meta, - ) + + if args.resume_from: + cfg.load_from = args.resume_from + cfg.resume = True + elif args.load_from: + cfg.load_from = args.load_from + cfg.resume = False + elif args.auto_resume: + cfg.resume = True + + if args.no_validate: + cfg.val_dataloader = None + cfg.val_cfg = None + cfg.val_evaluator = None + + if args.auto_scale_lr and isinstance(cfg.get("auto_scale_lr"), dict): + cfg.auto_scale_lr["enable"] = True + + if args.seed is not None: + seed = args.seed + if args.diff_seed: + seed += int(os.environ.get("LOCAL_RANK", "0")) + cfg.randomness = {"seed": seed, "deterministic": args.deterministic} + elif args.deterministic: + cfg.randomness = {"seed": None, "deterministic": True} + + if args.fp16: + optim_wrapper = cfg.get("optim_wrapper", {}) + if not isinstance(optim_wrapper, dict): + optim_wrapper = {} + optim_wrapper["type"] = "AmpOptimWrapper" + cfg.optim_wrapper = optim_wrapper + + cfg.launcher = args.launcher + + # Ensure visdet registry scope is active before building the runner. + DefaultScope.get_instance("visdet", scope_name="visdet") + + runner = Runner.from_cfg(cfg) + runner.train() if __name__ == "__main__": diff --git a/visdet/engine/model/utils.py b/visdet/engine/model/utils.py index 18feb194..66a1d284 100644 --- a/visdet/engine/model/utils.py +++ b/visdet/engine/model/utils.py @@ -208,15 +208,12 @@ def revert_sync_batchnorm(module: nn.Module) -> nn.Module: def convert_sync_batchnorm(module: nn.Module, implementation="torch") -> nn.Module: """Helper function to convert all `BatchNorm` layers in the model to - `SyncBatchNorm` (SyncBN) or `mmcv.ops.sync_bn.SyncBatchNorm` (MMSyncBN) - layers. Adapted from `PyTorch convert sync batchnorm`_. + `SyncBatchNorm` (SyncBN) layers. Args: module (nn.Module): The module containing `SyncBatchNorm` layers. - implementation (str): The type of `SyncBatchNorm` to convert to. - - - 'torch': convert to `torch.nn.modules.batchnorm.SyncBatchNorm`. - - 'mmcv': convert to `mmcv.ops.sync_bn.SyncBatchNorm`. + implementation (str): SyncBatchNorm implementation. + Only ``torch`` is supported. Returns: nn.Module: The converted module with `SyncBatchNorm` layers. @@ -226,15 +223,11 @@ def convert_sync_batchnorm(module: nn.Module, implementation="torch") -> nn.Modu """ module_output = module + if implementation != "torch": + raise ValueError(f'Only sync_bn="torch" is supported, but got {implementation}') + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm): - if implementation == "torch": - SyncBatchNorm = torch.nn.modules.batchnorm.SyncBatchNorm - elif implementation == "mmcv": - # MMCV is not a dependency in visdet; keep this option as an alias - # for backwards compatibility with upstream configs. - SyncBatchNorm = torch.nn.modules.batchnorm.SyncBatchNorm - else: - raise ValueError(f'sync_bn should be "torch" or "mmcv", but got {implementation}') + SyncBatchNorm = torch.nn.modules.batchnorm.SyncBatchNorm module_output = SyncBatchNorm( module.num_features, diff --git a/visdet/engine/utils/dl_utils/misc.py b/visdet/engine/utils/dl_utils/misc.py index 6070012b..7607fc10 100644 --- a/visdet/engine/utils/dl_utils/misc.py +++ b/visdet/engine/utils/dl_utils/misc.py @@ -1,7 +1,6 @@ # ruff: noqa # type: ignore # Copyright (c) OpenMMLab. All rights reserved. -import pkgutil import numpy as np import torch @@ -95,17 +94,3 @@ def has_batch_norm(model: nn.Module) -> bool: if has_batch_norm(m): return True return False - - -def mmcv_full_available() -> bool: - """Check whether mmcv-full is installed. - - Returns: - bool: True if mmcv-full is installed else False. - """ - try: - import mmcv # noqa: F401 - except ImportError: - return False - ext_loader = pkgutil.find_loader("mmcv._ext") - return ext_loader is not None