#!/bin/bash
# submit_mace_predictions.bsub ,  Re-run MACE seed 0 on the 4 scalar
# QM9 targets *with the prediction-saving patch in train_baseline_mace.py*.
# The first MACE pass produced the pooled R²/MAE numbers; this pass exists
# only so we can run isomer_audit.py against MACE.
#
# Array 1..4 = (gap, alpha, mu, zpve), seed=0 only.

#BSUB -J starg_mace_pred[1-4]
#BSUB -o logs/mace_pred_%I_%J.out
#BSUB -e logs/mace_pred_%I_%J.err
#BSUB -q normal
#BSUB -n 16
#BSUB -gpu "num=1:mode=exclusive_process"
#BSUB -W 24:00
#BSUB -M 128GB

set -uo pipefail
export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=1
mkdir -p logs results

TARGETS=(gap alpha mu zpve)
TARGET="${TARGETS[$(( LSB_JOBINDEX - 1 ))]}"
SEED=0

WORKDIR=$HOME/starg/python/large_scale
QM9_DIR=${QM9_DIR:-/u/$USER/data/qm9/dsgdb9nsd}

cd "$WORKDIR"
export PYTHONPATH=".:${PYTHONPATH:-}"

echo "[$(date)] host=$(hostname) array=$LSB_JOBINDEX target=$TARGET seed=$SEED"
nvidia-smi --query-gpu=name,memory.total --format=csv,noheader || true

python3 train_baseline_mace.py \
    --target  "$TARGET" \
    --qm9_dir "$QM9_DIR" \
    --seed    "$SEED" \
    --out_dir results/ \
    --device  cuda

echo "[$(date)] done"