#!/bin/bash
source config_analysis.sh
mkdir -p logs

log_message() {
  echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a logs/workflow.log
}

# Step 1: Submit aggregation jobs
log_message "STEP 1: Submitting parallel aggregation jobs (64 jobs)"
job_id=$(sbatch --parsable --array=1-64 run_aggregate.sh)

if [ $? -ne 0 ]; then
  log_message "ERROR: Failed to submit aggregation jobs"
  exit 1
fi

log_message "Aggregation jobs submitted: Job ID $job_id"
log_message ""

# Step 2: Submit copy job that depends on aggregation completion
log_message "STEP 2: Submitting RDS collection job (runs after aggregation)"
copy_job_id=$(sbatch --parsable --dependency=afterok:$job_id copy_rds_parallel.sh)

if [ $? -ne 0 ]; then
  log_message "ERROR: Failed to submit copy job"
  exit 1
fi