#!/bin/sh
#SBATCH --job-name=preciceTest
#SBATCH --partition=ckpt
#SBATCH --account=cesg
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=40 # each cesg group node has 40 total "cores" (i.e. 2 threads per 20 slots), but the UW Hyak Wiki recommends using 28 to run jobs most efficiently
#SBATCH --mem=120G
#SBATCH --time=2:00:00
#SBATCH --mail-type=ALL
#SBATCH --mail-user=nsaoirse@uw.edu

echo determining parallel processing parameters...
NPROC=$SLURM_NTASKS
echo $NPROC>NPROCRUN
# HYAK
module load cesg/openfoam/2012
module load cesg/eigen/3.3.9
module load cesg/libxml2/2.9.10
module load gcc/10.2.0
module load cesg/python/3.8.10
module load cesg/boost/1.76.0
module load cesg/openmpi/4.0.5
module load cmake/3.20.0


# TACC
# module load intel/19.1.1 
# module load impi/19.0.9    
# module load git/2.24.1    
# module load autotools/1.2   
# module load cmake/3.24.2   
# module load pmix/3.1.4     
# module load hwloc/1.11.12 
# module load xalt/2.10.34   
# module load TACC         
# module load python3/3.9.2 
# module load petsc/3.17   
# module load eigen/3.4.0  
# module load fftw3/3.3.10  
# module load boost/1.72

# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"/work2/07720/nsaoirse/frontera/lib/lib64:/work2/07720/nsaoirse/frontera/lib:/work2/07720/nsaoirse/frontera/source/OpenFOAM-v2106/nsaoirse-v2106/platforms/linux64GccDPInt32Opt/lib"
# export PATH=$PATH:"/work2/07720/nsaoirse/frontera/source/OpenFOAM-v2106/nsaoirse-v2106/platforms/linux64GccDPInt32Opt/bin:/work2/07720/nsaoirse/frontera/source/precice/src/precice"
# source /work2/07720/nsaoirse/frontera/source/OpenFOAM-v2106/etc/bashrc
# export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:"/work2/07720/nsaoirse/frontera/lib/lib64/pkgconfig"
. $WM_PROJECT_DIR/bin/tools/RunFunctions    # Tutorial run functions

# This script prepares and runs all the participants in one terminal,
# forwarding the solvers' output to log files.
# Alternatively, you may execute the scripts "runSolid" and "runFluid"
# in separate terminals.

python3 configuration_file.py

# Run this script with "-parallel" for parallel simulations

# The script "Allclean" cleans-up the result and log files.
# Set up the run parameters:

parallel=1
# =============== Participant 2: Solid ===========================

solverroot="./"
Participant2="FOAMySeesCouplingDriver"


configfile="precice-config.xml"



    # You can use CalculiX CGX to setup the structural simulation from sratch.
    # This will re-generate the all.msh, fix1.nam, surface.nam files.
    #
    # # Prepare in silent mode
    # echo "Preparing the ${Participant2} participant..."
    # cd ${Participant2}
    #     echo "  Executing cgx (provided by CalculiX, make sure this exists)..."
    #     cgx -bg pre_flap.fbd > prepare_flap.log 2>&1
    # cd ..

    # Run
    echo "Starting ${Participant2} participant..."
    mpirun -np 1 python3 ${solverroot}${Participant2}.py ${solverroot}${configfile} > ${Participant2}.log 2>&1 &
    PIDParticipant2=$!
	
	
sleep 20 
	
# =============== PREPARE Participant 1: Fluid ===========================
Participant1="Fluid"

# Prepare
echo "Preparing the ${Participant1} participant..."

cp Solid/interface.stl	Fluid/constant/triSurface

cd $Participant1 



		echo "  Preparing the mesh..."

		
		echo determining parallel processing parameters...


		echo preparing 0 folder...
		if [ -d '0' ]; then
		rm -r 0
		fi
		cp -r 0.org 0
		
		echo blockMesh meshing...
		blockMesh > log.blockMesh

		echo surfaceFeatureExtract extracting...
		surfaceFeatureExtract > log.surfFeatExt

        echo snappyHexMesh #I've been having trouble with parallizing this on TACC 
		
		#snappy hex mesh gives me segmentation faults seemingly at random
		#will probably need to recompile OpenFOAM at some point to fix this, but right now i can't manage the task
		
		# snappyHexMesh -overwrite -fileHandler uncollated > log.sHM
		echo decomposePar setting up parallel case...
		decomposePar -force -fileHandler uncollated -copyZero > log.decomp_sHM

		echo snappyHex meshing flume floor and/or structure planes...
		mpirun -np $NPROC snappyHexMesh -parallel -overwrite -fileHandler uncollated > log.sHM

		echo checking mesh quality...
		mpirun -np $NPROC checkMesh -parallel > log.checkMesh

		echo reconstructParMesh rebuilding mesh...
		reconstructParMesh -constant -mergeTol 1e-06 > log.reconMesh_sHM

		echo reconstructPar rebuilding fields...
		reconstructPar > log.recon_sHM

		echo eliminate meshing time step and processor directories...
		rm -r processor*
		
		# echo Setting the fields...
		# setFields > log.setFields
		
		cd ../..
		
		cd Fluid
		     mapFields -sourceTime 11.5 -case ../RunCase/Fluid .
		cd ../RunCase/Fluid

		mv 0/pointDisplacement.unmapped 0/pointDisplacement

		# setFields -case ${Participant1} > ${Participant1}_setFields.log 2>&1

		# get application information
	

		echo decomposePar setting up parallel case...

		echo Mesh built, ICs set


		Solver1=$(getApplication)    # solver
		echo "  ${Participant1} Solver: ${Solver1}."
    cd ..
	surfaceMeshExtract -case Fluid -patches 'interface' -latestTime FluidCouplingSurface.obj
    cp Fluid/FluidCouplingSurface.obj .
	
    # Run and get the process id
    if [ $parallel -eq 1 ]; then
        echo "  Decomposing the case..."
        decomposePar -fileHandler collated -force -case ${Participant1} > ${Participant1}_decomposePar.log 2>&1
        cd ${Participant1}
            nproc=$(getNumberOfProcessors)
        cd ..
        echo "parallel decomposition complete.  Starting the ${Participant1} participant in parallel..."
        mpirun -np ${nproc} ${Solver1} -parallel -fileHandler collated -case ${Participant1} > ${Participant1}.log 2>&1 &
    else
        echo "  Starting the ${Participant1} participant in serial..."
        ${Solver1} -case ${Participant1} > ${Participant1}.log 2>&1 &
    fi
    PIDParticipant1=$!


# =============== Wait for all the participants to finish =======
echo "Waiting for the participants to exit..., PIDs: ${PIDParticipant1}, ${PIDParticipant2}"
echo "(you may run 'tail -f ${Participant1}.log' in another terminal to check the progress)"

echo "To interrupt the simulation, press 'c'. Ctrl+C will only send the processes to the background."
while [ -e /proc/${PIDParticipant1} ]; do
    read -r -t1 -n1 input
    if [ "$input" = "c" ]; then
        kill ${PIDParticipant1}
        kill ${PIDParticipant2}
        false
    fi
done

if [ $? -ne 0 ] || [ "$(grep -c -E "error:" ${Participant1}.log)" -ne 0 ] || [ "$(grep -c -E "error:" ${Participant2}.log)" -ne 0 ]; then
    echo ""
    echo "Something went wrong... See the log files for more."
    # Precaution
    kill ${PIDParticipant1}
    kill ${PIDParticipant2}
else
    echo ""
    echo "The simulation completed! (check for any errors)"
    if [ $parallel -eq 1 ]; then
        echo "Reconstructing fields..."
        reconstructPar -case ${Participant1} > ${Participant1}_reconstructPar.log 2>&1 & 
    fi
	wait 
	foamToVTK -case Fluid
	wait
	
	python3 FSIPVD.py
	
	wait
	
	mkdir results
	
	cp -r SeesOutput results
	cp -r OpenSeesOutput.pvd results
	cp -r Fluid/VTK results
	cp -r Fluid/postProcessing results
	
	zip -r ../results.zip results
	
    echo "You may now open '${Participant1}/${Participant1}.foam' in ParaView."
    # Note: ".foam" triggers the native OpenFOAM reader of ParaView.
    # Change to ".OpenFOAM" to use the OpenFOAM reader provided with OpenFOAM.
fi

echo ""
echo "### NOTE ### Make sure to use the correct solver for your OpenFOAM version! (pimpleFoam for OpenFOAM v1806, OpenFOAM 6, or newer, vs pimpleDyMFoam for older) You may change this in your Fluid/system/controlDict file, if needed."
