kub-dataset: Full Reference

kub-dataset pack <location> [options]
kub-dataset unpack <archive> [options]
kub-dataset list <archive>
kub-dataset push <location> --version <version> --api-key <key> [--dmp <dmp>]
kub-dataset pull <location> [--version <version>] [--dmp <dmp>] [--dvc]
kub-dataset delete <location> [--version <version>] --api-key <key>
kub-dataset list-versions <location> [--dmp <dmp>]
kub-dataset list-locations [--dmp <dmps>] [--show-versions]
kub-dataset summary [<location> ...] [--all] [--version <version>] [--format {table,json}]
kub-dataset list-dmps
kub-dataset generate {meteo|iaq|both} <location> --start <date|datetime> --end <date|datetime> [--version <version>]
kub-dataset manifest-show <location> [--version <version>]
kub-dataset manifest-regenerate <location> [--version <version>]
kub-dataset manifest-verify <location> [--version <version>]
kub-dataset manifest-validate [manifest|location] [--version <version>] [--cemdb-root <path>] [--strict]
kub-dataset fix <location> [--version <version>] [--policy-file <path>] [--dry-run]
kub-dataset policy list
kub-dataset policy show [policy_ref] [--template <name>|--file <path>]
kub-dataset policy validate [policy_ref] [--template <name>|--file <path>] [-v]
kub-dataset policy apply <location> [--version <version>] [--template <name>|--file <path>] [--dry-run]
kub-dataset policy-validate <policy-file> [-v]
kub-dataset migrate [<location>] [--execute]
kub-dataset migrate-layout [<location>|--all]
kub-dataset migrate-org-layout [<location>|--all] [--organization <org>|--org-map <json>]
kub-dataset copy-version <location> <source-version> <target-version>
kub-dataset rename-version <location> <old-version> <new-version>
kub-dataset rename-location <old-name> <new-name>
kub-dataset set-current <location> <version>
kub-dataset duplicate <location> [<source-version>]
kub-dataset tag-reference <location> <run-id> [--type <single|ensemble>]
kub-dataset list-reference <location> [--type <single|ensemble>]
kub-dataset push-component <location> --version <version> --component <geo|config|inputs|preprocessing|all> --api-key <key>
kub-dataset pull-component <location> --version <version> --component <geo|config|inputs|preprocessing>
kub-dataset list-components <location> --version <version>

kub-dataset pack-simulator --version <version> [--lod <lod>] [--single]
kub-dataset unpack-simulator <archive> [options]
kub-dataset push-simulator --version <version> [--lod <lod>] --api-key <key>
kub-dataset pull-simulator --version <version> [--lod <lod>] [--dvc]
kub-dataset list-simulators [--local] [--remote] [--version <version>]
kub-dataset delete-simulator --version <version> [--lod <lod>] --api-key <key>

kub-dataset init [--path <path>]
kub-dataset status [--path <path>]
kub-dataset login --provider <provider-id>
kub-dataset logout [--provider <provider-id>|--all]
kub-dataset whoami

cemdb/simulators/
└── v0.1.0/                         # Version folder
    ├── manifest.json               # Version manifest (describes all directories)
    ├── lod0/                       # LOD 0 building models
    │   ├── App4Walls1Floor1Roof.fmu
    │   ├── App4Walls2Floor1RoofBoiler.fmu
    │   └── ... (60 FMU files)
    ├── lod0.metadata.json          # LOD 0 metadata (building model details)
    ├── others/                     # Support FMU files
    │   └── Sun.fmu
    └── others.metadata.json        # Support files metadata

UrbanBuilding/cemdb/simulators/
└── v0.1.0/
    ├── manifest.json               # Manifest item
    ├── lod0                        # Item containing all LOD0 FMU files
    └── others                      # Item containing support FMU files

{
    "version": "0.1.0",
    "description": "Dataset of physical building models in FMU format",
    "creationDate": "2025-10-22",
    "itemType": "Simulators",
    "maxNumberOfFloors": 10,
    "totalNumberOfBuildingModels": 60,
    "directories": [
        {
            "name": "lod0",
            "type": "lod",
            "lod": 0,
            "description": "Level of Detail 0 building models",
            "count": 60
        },
        {
            "name": "others",
            "type": "support",
            "description": "Supporting FMU files (Sun model, etc.)",
            "files": ["Sun.fmu"]
        }
    ]
}

# 1. Initialize DVC (one-time setup)
kub-dataset init

# 2. Pull dataset with DVC tracking
kub-dataset pull kernante --version 0.98.0 --dvc

# 3. Check tracked datasets
kub-dataset status

# 4. Commit to git
git add .dvc .dvcignore cemdb/locations/kernante.dvc cemdb/locations/.gitignore
git commit -m "Track kernante v0.98.0 with DVC"

# DVC tracking file for kernante
# Generated by kub-dataset
# DO NOT EDIT - managed by kub-dataset pull --dvc
outs:
- md5: a1b2c3d4e5f6...
  path: kernante
  hash: md5
  size: 183456
  nfiles: 42
meta:
  source: girder-unistra
  version: 0.98.0
  tool: kub-dataset

# 1. Initialize DVC in the repository (one-time)
kub-dataset init
git add .dvc .dvcignore
git commit -m "Initialize DVC"

# 2. Pull dataset with DVC tracking
kub-dataset pull kernante --version 0.98.0 --dvc

# 3. Verify tracking
kub-dataset status
# Output:
# Tracked datasets:
#   - kernante (version: 0.98.0, source: girder-unistra)

# 4. Commit the .dvc file to git
git add cemdb/locations/kernante.dvc cemdb/locations/.gitignore
git commit -m "Track kernante v0.98.0"

# 5. Later, anyone can reproduce the exact dataset
kub-dataset pull kernante --version 0.98.0 --dvc --force

# Update to a new version
kub-dataset pull kernante --version 1.0.0 --dvc --force

# Check the updated version
kub-dataset status
# Output:
# Tracked datasets:
#   - kernante (version: 1.0.0, source: girder-unistra)

# Commit the change
git add cemdb/locations/kernante.dvc
git commit -m "Update kernante to v1.0.0"

# GitHub Actions example
- name: Install kub-dataset
  run: pip install feelpp-ktirio-urban-building

- name: Pull dataset
  run: |
    kub-dataset pull kernante \
      --cemdb-root ./cemdb/locations \
      --api-key "${{ secrets.GIRDER_API_KEY }}" \
      --force -v

# Clone repository
git clone https://github.com/feelpp/ktirio-urban-building.git
cd ktirio-urban-building

# List available datasets
kub-dataset list-locations

# Pull required datasets with DVC tracking
kub-dataset pull kernante --dvc --force
kub-dataset pull lingolsheim --dvc --force

# Verify downloaded data
ls -la cemdb/locations/

# Verify local dataset is ready
ls cemdb/locations/kernante/

# Push new version to Girder
kub-dataset -v push kernante \
  --version 1.0.0 \
  --api-key $GIRDER_API_KEY \
  --cemdb-root cemdb/locations

# Verify it's available
kub-dataset list-versions kernante

# Pull from Girder
kub-dataset pull kernante \
  --dmp girder-unistra \
  --api-key $GIRDER_API_KEY \
  --cemdb-root ./temp

# Push to CKAN
kub-dataset push kernante \
  --version 1.0.0 \
  --dmp ckan-hidalgo2 \
  --api-key $CKAN_API_KEY \
  --cemdb-root ./temp

# Initial setup: pull with DVC tracking
kub-dataset pull kernante --version 0.98.0 --dvc

# Commit the .dvc file
git add cemdb/locations/kernante.dvc
git commit -m "Track kernante v0.98.0"

# Later: reproduce the exact dataset
# (read version from .dvc file's meta section)
cat cemdb/locations/kernante.dvc | grep version

# Pack location for distribution
kub-dataset -v pack kernante \
  --cemdb-root cemdb/locations \
  -o kernante_test_input.zip

# Verify archive contents
kub-dataset list kernante_test_input.zip | head -20

# Share or upload the archive
# Recipients can unpack with:
# kub-dataset unpack kernante_test_input.zip --force

# Get overview of all available data
kub-dataset list-locations --show-versions

# Export to JSON for processing
kub-dataset list-locations --format json > datasets.json

# Query specific DMPs with verbose output
kub-dataset -v list-locations --dmp girder-unistra

from feelpp.ktirio.ub.dataset import (
    pack_dataset,
    unpack_dataset,
    pull_dataset_girder,
    pull_dataset_ckan,
    push_dataset_girder,
    delete_dataset_girder,
    list_locations_all,
    list_versions_girder,
    generate_dvc_file,
    init_dvc,
    status_dvc,
)

# Initialize DVC
init_dvc(verbose=True)

# Pack a location
zip_path = pack_dataset("kernante", cemdb_root="cemdb/locations")

# Pull from Girder
location_path = pull_dataset_girder(
    "kernante",
    version="0.98.0",
    api_key="your-api-key",
    cemdb_root="cemdb/locations",
)

# Generate DVC tracking
dvc_path = generate_dvc_file(
    location_path,
    dmp_id="girder-unistra",
    version="0.98.0",
)

# Check DVC status
result = status_dvc()
print(f"Initialized: {result['initialized']}")
print(f"Status: {result['status']}")

# List all locations
locations = list_locations_all(verbose=True)
for loc in locations:
    print(f"{loc['dmp']}: {loc['name']} v{loc['latest_version']}")

from feelpp.ktirio.ub.dataset import (
    # Pack operations
    pack_simulator_lod,
    pack_simulator_version,
    unpack_simulator_lod,
    unpack_simulator_version,
    list_simulator_lods_local,
    list_simulator_versions_local,
    # DMP operations (Girder only)
    push_simulator_lod_girder,
    push_simulator_manifest_girder,
    pull_simulator_lod_girder,
    list_simulator_lods_girder,
    delete_simulator_lod_girder,
)

# Pack a specific LOD
zip_path = pack_simulator_lod(
    version="0.1.0",
    lod=0,  # Can also be 'others'
    cemdb_root="cemdb/simulators",
)

# Pack entire version (single zip with manifest)
zip_path = pack_simulator_version(
    version="0.1.0",
    cemdb_root="cemdb/simulators",
)

# Unpack a single directory zip (lod0 or others)
dir_path, version, dir_name = unpack_simulator_lod(
    "simulators_lod0-v0.1.0.zip",
    cemdb_root="cemdb/simulators",
)
print(f"Unpacked {dir_name} to {dir_path}")

# Unpack a unified version zip (all directories + manifest)
version_path, version = unpack_simulator_version(
    "simulators-v0.1.0.zip",
    cemdb_root="cemdb/simulators",
)
print(f"Unpacked version {version} to {version_path}")

# List local versions
versions = list_simulator_versions_local(cemdb_root="cemdb/simulators")
print(f"Local versions: {versions}")

# List directories in a version
dirs = list_simulator_lods_local("0.1.0", cemdb_root="cemdb/simulators")
print(f"Directories: {dirs}")  # [0, 'others']

# Pull from Girder
dir_path, version, dir_name = pull_simulator_lod_girder(
    lod=0,
    version="0.1.0",
    api_key="your-api-key",
    cemdb_root="cemdb/simulators",
)
print(f"Pulled {dir_name} to {dir_path}")

# Push to Girder
result = push_simulator_lod_girder(
    version="0.1.0",
    lod=0,
    api_key="your-api-key",
    cemdb_root="cemdb/simulators",
)
print(f"Uploaded: {result['item_name']}")

# Push manifest
result = push_simulator_manifest_girder(
    version="0.1.0",
    api_key="your-api-key",
    cemdb_root="cemdb/simulators",
)

# List remote versions
remote_data = list_simulator_lods_girder(verbose=True)
for version_info in remote_data:
    print(f"v{version_info['version']}: {version_info['item_count']} items")

from feelpp.ktirio.ub.dataset.generators import bestest

# Generate BESTEST reference cases
bestest.generate_bestest_dataset(output_dir="cemdb/locations/bestest")

from feelpp.ktirio.ub.dataset.sources import openmeteo, events

# Fetch IAQ data for simulation
from feelpp.ktirio.ub.dataset.sources.openmeteo import fetch_iaq_for_simulation

# Fetch historical weather for event scenarios
from feelpp.ktirio.ub.dataset.sources.events import fetch_event_to_location

fetch_event_to_location(
    location_root="cemdb/locations/notre_dame/v0.1.0",
    event_name="fire-2019-04-15",
    lat=48.853, lon=2.349,
    start="2019-04-15", end="2019-04-16",
)