Commit 674d7f4b authored by Marcel Henrik Schubert's avatar Marcel Henrik Schubert
Browse files

batch scripts

parent e0d6d570
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$/.." vcs="Git" />
</component>
</project>
\ No newline at end of file
#!/bin/bash -l
# Standard output and error:
# #SBATCH --open-mode=truncate
#SBATCH -o ./out/Preprocessing/preproc_%j.out
#SBATCH -e ./out/Preprocessing/preproc_%j.err
# Initial working directory:
#SBATCH -D /ptmp/mschuber/PAN/Preprocessing
# Job Name:
#SBATCH -J preprocess
# Queue:
#SBATCH --partition=medium
# Number of nodes and MPI tasks per node:
#SBATCH --nodes=3
#SBATCH --ntasks-per-node=1
# Enable Hyperthreading:
# #SBATCH --ntasks-per-core=2
# for OpenMP:
#SBATCH --cpus-per-task=80
#SBATCH --mail-type=none
#SBATCH --mail-user=schubert@coll.mpg.de
# Wall clock limit:
#SBATCH --time=24:00:00
##export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
# For pinning threads correctly:
##export OMP_PLACES=cores
# Set the number of OMP threads *per process* to avoid overloading of the node!
export OMP_NUM_THREADS=1
module purge
module load gcc/8
module load anaconda/3/2020.02
module load tensorflow/cpu/2.5.0
module load scikit-learn/0.24.1
##bash script foo multinode processing
names=(creator performer sports)
for i in ${names[@]}; do
# Run the program:
srun -N 1 -n 1 python preprocess.py -p ../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31 -f workset_$i.ndjson -s ../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31/preprocessed -c (1,5) -w (1,2) -t (1,3) -d (1,5) -o (1,3) --workset=workset --part=$i --rerun --both --asis --spacy --encase_list emoji emoticon
done
wait
echo "job finished"
!/bin/bash -l
typ=creator
# Standard output and error:
#SBATCH -o ./../jobscripts/out/preprocess_${typ}_out.%j
#SBATCH -e ./../jobscripts/out/preprocess_${typ}_err_part_2.%j
# Initial working directory:
#SBATCH -D /ptmp/mschuber/PAN/Scripts/Preprocessing
# Job Name:
#SBATCH -J preprocess_${typ}
# Queue:
#SBATCH --partition=medium
# Number of nodes and MPI tasks per node:
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
# Enable Hyperthreading:
##SBATCH --ntasks-per-core=2
# for OpenMP:
#SBATCH --cpus-per-task=80
#
#SBATCH --mem=180000
#SBATCH --mail-type=none
#SBATCH --mail-user=schubert@coll.mpg.de
# Wall clock limit:
#SBATCH --time=24:00:00
##export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
# For pinning threads correctly:
##export OMP_PLACES=cores
# Set the number of OMP threads *per process* to avoid overloading of the node!
export OMP_NUM_THREADS=1
module purge
module load gcc/8
module load anaconda/3/2020.02
module load tensorflow/cpu/2.5.0
module load scikit-learn/0.24.1
python preprocess.py -p ../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31 -f workset_${typ}.ndjson -s ../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31/preprocessed -c "(1,5)" -w "(1,2)" -t "(1,3)" -d "(1,5)" -o "(1,3)" --workset=workset --part=${typ} --rerun --both --asis --spacy --encase_list emoji emoticon
echo "job finished"
\ No newline at end of file
!/bin/bash -l
typ=manager
# Standard output and error:
#SBATCH -o ./../jobscripts/out/preprocess_${typ}_out.%j
#SBATCH -e ./../jobscripts/out/preprocess_${typ}_err_part_2.%j
# Initial working directory:
#SBATCH -D /ptmp/mschuber/PAN/Scripts/Preprocessing
# Job Name:
#SBATCH -J preprocess_${typ}
# Queue:
#SBATCH --partition=medium
# Number of nodes and MPI tasks per node:
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
# Enable Hyperthreading:
##SBATCH --ntasks-per-core=2
# for OpenMP:
#SBATCH --cpus-per-task=80
#
#SBATCH --mem=180000
#SBATCH --mail-type=none
#SBATCH --mail-user=schubert@coll.mpg.de
# Wall clock limit:
#SBATCH --time=24:00:00
##export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
# For pinning threads correctly:
##export OMP_PLACES=cores
# Set the number of OMP threads *per process* to avoid overloading of the node!
export OMP_NUM_THREADS=1
module purge
module load gcc/8
module load anaconda/3/2020.02
module load tensorflow/cpu/2.5.0
module load scikit-learn/0.24.1
python preprocess.py -p ../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31 -f workset_${typ}.ndjson -s ../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31/preprocessed -c "(1,5)" -w "(1,2)" -t "(1,3)" -d "(1,5)" -o "(1,3)" --workset=workset --part=${typ} --rerun --both --asis --spacy --encase_list emoji emoticon
echo "job finished"
\ No newline at end of file
!/bin/bash -l
typ=performer
# Standard output and error:
#SBATCH -o ./../jobscripts/out/preprocess_${typ}_out.%j
#SBATCH -e ./../jobscripts/out/preprocess_${typ}_err_part_2.%j
# Initial working directory:
#SBATCH -D /ptmp/mschuber/PAN/Scripts/Preprocessing
# Job Name:
#SBATCH -J preprocess_${typ}
# Queue:
#SBATCH --partition=medium
# Number of nodes and MPI tasks per node:
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
# Enable Hyperthreading:
##SBATCH --ntasks-per-core=2
# for OpenMP:
#SBATCH --cpus-per-task=80
#
#SBATCH --mem=180000
#SBATCH --mail-type=none
#SBATCH --mail-user=schubert@coll.mpg.de
# Wall clock limit:
#SBATCH --time=24:00:00
##export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
# For pinning threads correctly:
##export OMP_PLACES=cores
# Set the number of OMP threads *per process* to avoid overloading of the node!
export OMP_NUM_THREADS=1
module purge
module load gcc/8
module load anaconda/3/2020.02
module load tensorflow/cpu/2.5.0
module load scikit-learn/0.24.1
python preprocess.py -p ../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31 -f workset_${typ}.ndjson -s ../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31/preprocessed -c "(1,5)" -w "(1,2)" -t "(1,3)" -d "(1,5)" -o "(1,3)" --workset=workset --part=${typ} --rerun --both --asis --spacy --encase_list emoji emoticon
echo "job finished"
\ No newline at end of file
!/bin/bash -l
typ=sports
# Standard output and error:
#SBATCH -o ./../jobscripts/out/preprocess_${typ}_out.%j
#SBATCH -e ./../jobscripts/out/preprocess_${typ}_err_part_2.%j
# Initial working directory:
#SBATCH -D /ptmp/mschuber/PAN/Scripts/Preprocessing
# Job Name:
#SBATCH -J preprocess_${typ}
# Queue:
#SBATCH --partition=medium
# Number of nodes and MPI tasks per node:
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
# Enable Hyperthreading:
##SBATCH --ntasks-per-core=2
# for OpenMP:
#SBATCH --cpus-per-task=80
#
#SBATCH --mem=180000
#SBATCH --mail-type=none
#SBATCH --mail-user=schubert@coll.mpg.de
# Wall clock limit:
#SBATCH --time=24:00:00
##export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK}
# For pinning threads correctly:
##export OMP_PLACES=cores
# Set the number of OMP threads *per process* to avoid overloading of the node!
export OMP_NUM_THREADS=1
module purge
module load gcc/8
module load anaconda/3/2020.02
module load tensorflow/cpu/2.5.0
module load scikit-learn/0.24.1
python preprocess.py -p ../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31 -f workset_${typ}.ndjson -s ../../Data/pan19-celebrity-profiling-training-dataset-2019-01-31/preprocessed -c "(1,5)" -w "(1,2)" -t "(1,3)" -d "(1,5)" -o "(1,3)" --workset=workset --part=${typ} --rerun --both --asis --spacy --encase_list emoji emoticon
echo "job finished"
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment