Armin Seimel
77 lines
2.9 KiB
Bash
Executable File
77 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
|
cd "$repo_root"
|
|
|
|
if [[ "${PARTY2D_APPROVE_MODEL_FIT:-}" != "YES" ]]; then
|
|
echo "Refusing to start a Stan fit without explicit approval." >&2
|
|
echo "After approval, rerun with PARTY2D_APPROVE_MODEL_FIT=YES." >&2
|
|
exit 64
|
|
fi
|
|
|
|
run_dir="${1:-$repo_root/_local/validation/blocked_party}"
|
|
mkdir -p "$run_dir" "$repo_root/_local/tmp/blocked_validation"
|
|
|
|
if pgrep -af 'stan_model_2dim_v6.*sample|src/julia/01_run_model.jl' | grep -v "$$" >/dev/null; then
|
|
echo "Refusing to start: another party2d model process appears to be running." >&2
|
|
pgrep -af 'stan_model_2dim_v6.*sample|src/julia/01_run_model.jl' >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -f "$run_dir/model_fit.complete" ]]; then
|
|
echo "Blocked validation already marked complete: $run_dir/model_fit.complete" >&2
|
|
exit 1
|
|
fi
|
|
|
|
required_kb=$((80 * 1024 * 1024))
|
|
available_kb="$(df -Pk "$run_dir" | awk 'NR==2 {print $4}')"
|
|
if (( available_kb < required_kb )); then
|
|
echo "Refusing to start: less than 80 GiB free on the run volume." >&2
|
|
exit 1
|
|
fi
|
|
|
|
julia --project=. validation/prepare_blocked_validation.jl "$run_dir"
|
|
sha256sum "$run_dir"/text_data.csv "$run_dir"/expert.csv "$run_dir"/lr_data.csv \
|
|
"$run_dir"/expert_test.csv "$run_dir"/lr_data_test.csv \
|
|
> "$run_dir/input_sha256sums.txt"
|
|
|
|
export CMDSTAN_HOME="${CMDSTAN_HOME:-/opt/agent-tools/cmdstan-2.39.0}"
|
|
export TMPDIR="$repo_root/_local/tmp/blocked_validation"
|
|
export JULIA_NUM_THREADS="${JULIA_NUM_THREADS:-4}"
|
|
export STAN_NUM_THREADS=1
|
|
export STAN_REFRESH="${STAN_REFRESH:-50}"
|
|
export PARTY2D_NUM_CHAINS="${PARTY2D_NUM_CHAINS:-4}"
|
|
export PARTY2D_NUM_WARMUP="${PARTY2D_NUM_WARMUP:-1000}"
|
|
export PARTY2D_NUM_SAMPLES="${PARTY2D_NUM_SAMPLES:-1000}"
|
|
|
|
if [[ ! -x "$CMDSTAN_HOME/bin/stanc" ]]; then
|
|
echo "Refusing to start: no executable stanc compiler at $CMDSTAN_HOME/bin/stanc" >&2
|
|
exit 1
|
|
fi
|
|
"$CMDSTAN_HOME/bin/stanc" --version > "$run_dir/stanc_version.txt"
|
|
|
|
date --iso-8601=seconds > "$run_dir/model_fit.started"
|
|
echo "$$" > "$run_dir/model_fit.pid"
|
|
echo "CMDSTAN_HOME=$CMDSTAN_HOME" > "$run_dir/model_fit.environment"
|
|
echo "JULIA_NUM_THREADS=$JULIA_NUM_THREADS" >> "$run_dir/model_fit.environment"
|
|
echo "STAN_NUM_THREADS=$STAN_NUM_THREADS" >> "$run_dir/model_fit.environment"
|
|
echo "PARTY2D_NUM_CHAINS=$PARTY2D_NUM_CHAINS" >> "$run_dir/model_fit.environment"
|
|
echo "PARTY2D_NUM_WARMUP=$PARTY2D_NUM_WARMUP" >> "$run_dir/model_fit.environment"
|
|
echo "PARTY2D_NUM_SAMPLES=$PARTY2D_NUM_SAMPLES" >> "$run_dir/model_fit.environment"
|
|
|
|
set +e
|
|
nice -n 10 julia --project=. src/julia/01_run_model.jl --data-dir "$run_dir" \
|
|
2>&1 | tee "$run_dir/model_fit.log"
|
|
status="${PIPESTATUS[0]}"
|
|
set -e
|
|
|
|
echo "$status" > "$run_dir/model_fit.exit_code"
|
|
if [[ "$status" -eq 0 ]]; then
|
|
rm -f "$run_dir/model_fit.failed"
|
|
date --iso-8601=seconds > "$run_dir/model_fit.complete"
|
|
else
|
|
date --iso-8601=seconds > "$run_dir/model_fit.failed"
|
|
fi
|
|
exit "$status"
|