Merge pull request #108 from BerkeleyLab/create-icar-trainer-subdir

Add new fpm subproject `icar-trainer`

app/train-cloud-microphysics.f90 → cloud-microphysics/app/train-cloud-microphysics.f90

@@ -17,8 +17,9 @@ program train_cloud_microphysics
 
   !! Internal dependencies;
   use inference_engine_m, only : &
-    inference_engine_t, mini_batch_t, input_output_pair_t, tensor_t, trainable_engine_t, rkind, NetCDF_file_t, &
+    inference_engine_t, mini_batch_t, input_output_pair_t, tensor_t, trainable_engine_t, rkind, &
     training_configuration_t, shuffle
+  use NetCDF_file_m, only: NetCDF_file_t
   use ubounds_m, only : ubounds_t
   implicit none
 

cloud-microphysics/fpm.toml

@@ -0,0 +1,11 @@
+name = "icar-trainer"
+version = "0.0.0"
+license = "license"
+author = "Damian Rouson, Tan Nguyen, Jordan Welsman, David Torres, Brad Richardson, Katherine Rasmussen"
+maintainer = "rouson@lbl.gov"
+
+[dependencies]
+assert = {git = "https://github.com/sourceryinstitute/assert", tag = "1.5.0"}
+sourcery = {git = "https://github.com/sourceryinstitute/sourcery", tag = "4.5.0"}
+inference-engine = {path = "../"}
+netcdf-interfaces = {git = "https://github.com/LKedward/netcdf-interfaces.git", rev = "d2bbb71ac52b4e346b62572b1ca1620134481096"}

cloud-microphysics/setup.sh

@@ -0,0 +1,153 @@
+#!/bin/sh
+
+set -e # exit on error
+
+usage()
+{
+  echo "Inference Engine Setup Script"
+  echo ""
+  echo "USAGE:"
+  echo "./setup.sh [--help|-h] | [-p|--prefix=PREFIX]"
+  echo ""
+  echo " --help             Display this help text"
+  echo " --prefix=PREFIX    Install binary in 'PREFIX/bin'"
+  echo "                    Default prefix='\$HOME/.local/bin'"
+  echo ""
+}
+
+PREFIX="$HOME/.local"
+
+while [ "$1" != "" ]; do
+  PARAM=$(echo "$1" | awk -F= '{print $1}')
+  VALUE=$(echo "$1" | awk -F= '{print $2}')
+  case $PARAM in
+    -h | --help)
+      usage
+      exit
+      ;;
+    -p | --prefix)
+      PREFIX=$VALUE
+      ;;
+    *)
+      echo "ERROR: unknown parameter \"$PARAM\""
+      usage
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+set -u # error on use of undefined variable
+
+if ! command -v brew > /dev/null ; then
+  if ! command -v curl > /dev/null ; then
+    echo "Please install curl and then rerun ./setup.sh"
+    exit 1
+  fi
+  /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+  if [ $(uname) = "Linux" ]; then
+    if [ -z "$PATH" ]; then
+      PATH=/home/linuxbrew/.linuxbrew/bin/
+    else
+      PATH=/home/linuxbrew/.linuxbrew/bin/:"$PATH"
+    fi
+  fi
+fi
+
+brew tap fortran-lang/fortran # required for building fpm
+brew install fortran-lang/fortran/fpm netcdf netcdf-fortran pkg-config coreutils # coreutils supports `realpath` below
+
+PREFIX=`realpath $PREFIX`
+
+NETCDF_LIB_PATH="`brew --prefix netcdf`/lib"
+HDF5_LIB_PATH="`brew --prefix hdf5`/lib"
+NETCDFF_LIB_PATH="`brew --prefix netcdf-fortran`/lib"
+
+FPM_LD_FLAG=" -L$NETCDF_LIB_PATH -L$HDF5_LIB_PATH -L$NETCDFF_LIB_PATH"
+FPM_FLAG="-fcoarray=single -O3 -fallow-argument-mismatch -ffree-line-length-none -L$NETCDF_LIB_PATH -L$HDF5_LIB_PATH"
+FPM_FC=${FC:-"gfortran-13"}
+FPM_CC=${CC:-"gcc-13"}
+
+mkdir -p build
+
+CI=${CI:-"false"} # GitHub Actions workflows set CI=true
+
+if [ $CI = true ]; then
+  PKG_CONFIG_PATH=`realpath ./build/pkgconfig`
+  echo "---------------"
+  echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH"
+  echo "---------------"
+else
+  PKG_CONFIG_PATH=`realpath "$PREFIX"/lib/pkgconfig`
+fi
+
+if [ ! -d $PKG_CONFIG_PATH ]; then
+  mkdir -p $PKG_CONFIG_PATH 
+fi
+
+INFERENCE_ENGINE_PC="$PKG_CONFIG_PATH/inference-engine.pc"
+echo "INFERENCE_ENGINE_FPM_CC=\"$FPM_CC\""                  >  $INFERENCE_ENGINE_PC
+echo "INFERENCE_ENGINE_FPM_FC=\"$FPM_FC\""                  >> $INFERENCE_ENGINE_PC
+echo "INFERENCE_ENGINE_FPM_LD_FLAG=\"$FPM_LD_FLAG\""        >> $INFERENCE_ENGINE_PC
+echo "INFERENCE_ENGINE_FPM_FLAG=\"$FPM_FLAG\""              >> $INFERENCE_ENGINE_PC
+echo "Name: inference-engine"                               >> $INFERENCE_ENGINE_PC
+echo "Description: Inference Engine"                        >> $INFERENCE_ENGINE_PC
+echo "URL: https://github.com/berkeleylab/inference-engine" >> $INFERENCE_ENGINE_PC
+echo "Version: 0.1.2"                                       >> $INFERENCE_ENGINE_PC
+if [ $CI = true ]; then
+  echo "---------------"
+  echo "cat $INFERENCE_ENGINE_PC"
+  cat $INFERENCE_ENGINE_PC
+  echo "---------------"
+fi
+
+export PKG_CONFIG_PATH
+touch build/run-fpm.sh
+RUN_FPM_SH="`realpath ./build/run-fpm.sh`"
+echo "#!/bin/sh" >> $RUN_FPM_SH
+echo "#-- DO NOT EDIT -- created by inference-engine/setup.sh"  >> $RUN_FPM_SH
+echo "export PKG_CONFIG_PATH" >> $RUN_FPM_SH
+echo "" >> $RUN_FPM_SH
+echo "fpm_arguments=\"\"" >> $RUN_FPM_SH
+echo "program_arguments=\"\"" >> $RUN_FPM_SH
+echo "while test \$# -gt 0" >> $RUN_FPM_SH
+echo "do" >> $RUN_FPM_SH
+echo "    case \"\$1\" in" >> $RUN_FPM_SH
+echo "        --) program_arguments=\"\$@\"" >> $RUN_FPM_SH
+echo "            ;;" >> $RUN_FPM_SH
+echo "        *) if [ -z \"\$program_arguments\" ];  then" >> $RUN_FPM_SH
+echo "             fpm_arguments=\"\$fpm_arguments \$1\"" >> $RUN_FPM_SH
+echo "           fi" >> $RUN_FPM_SH
+echo "            ;;" >> $RUN_FPM_SH
+echo "    esac" >> $RUN_FPM_SH
+echo "    shift" >> $RUN_FPM_SH
+echo "done" >> $RUN_FPM_SH
+echo "" >> $RUN_FPM_SH
+echo "`which fpm` \$fpm_arguments \\" >>  $RUN_FPM_SH
+echo "--profile release \\" >> $RUN_FPM_SH
+echo "--c-compiler \"`pkg-config inference-engine --variable=INFERENCE_ENGINE_FPM_CC`\" \\" >> $RUN_FPM_SH
+echo "--compiler \"`pkg-config inference-engine --variable=INFERENCE_ENGINE_FPM_FC`\" \\" >> $RUN_FPM_SH
+echo "--flag \"-cpp `pkg-config inference-engine --variable=INFERENCE_ENGINE_FPM_FLAG`\" \\"  >> $RUN_FPM_SH
+echo "--link-flag \"`pkg-config inference-engine --variable=INFERENCE_ENGINE_FPM_LD_FLAG`\" \\" >> $RUN_FPM_SH
+echo "\$program_arguments" >> $RUN_FPM_SH
+chmod u+x $RUN_FPM_SH
+if [ $CI = true ]; then
+  echo "---------------"
+  echo "cat $RUN_FPM_SH"
+  cat $RUN_FPM_SH
+  echo "---------------"
+fi
+
+echo "$RUN_FPM_SH test"
+$RUN_FPM_SH test
+
+echo ""
+echo "____________________ cloud-microphysics has been set up! _______________________"
+echo ""
+echo "Usage:"
+echo ""
+echo "./build/run-fpm.sh run train-cloud-microphysics -- \ "
+echo "  --base <string> --epochs <integer> \ "
+echo "  [--start <integer>] [--end <integer>] [--stride <integer>]"
+echo ""
+echo "where angular brackets denote user-provided values and square brackets denote optional arguments"

cloud-microphysics/test/main.f90

@@ -0,0 +1,25 @@
+! Copyright (c), The Regents of the University of California
+! Terms of use are as specified in LICENSE.txt
+program main
+  use netCDF_file_test_m, only : netCDF_file_test_t
+  implicit none
+
+  real t_start, t_finish
+
+  integer :: passes=0, tests=0
+
+  call cpu_time(t_start)
+  block 
+    type(netCDF_file_test_t) netCDF_file_test
+    call netCDF_file_test%report(passes, tests)
+  end block
+  call cpu_time(t_finish)
+
+  print *
+  print *,"Test suite execution time: ",t_finish - t_start
+  print *
+  print '(*(a,:,g0))',"_________ In total, ",passes," of ",tests, " tests pass. _________"
+  sync all
+  print *
+  if (passes/=tests) error stop "-------- One or more tests failed. See the above report. ---------"
+end program

example/learn-exponentiation.f90

@@ -120,7 +120,7 @@ pure function e(j,n) result(unit_vector)
   function perturbed_identity_network(perturbation_magnitude) result(trainable_engine)
     type(trainable_engine_t) trainable_engine
     real, intent(in) :: perturbation_magnitude
-    integer, parameter :: n(*) = [8, 64, 64, 64, 6]
+    integer, parameter :: n(*) = [8, 64, 64, 64, 6] ! nodes per layer (first layer = input, last layer = output)
     integer, parameter :: n_max = maxval(n), layers = size(n)
     integer j, k, l
     real, allocatable :: identity(:,:,:), w_harvest(:,:,:), b_harvest(:,:)
@@ -144,4 +144,4 @@ function perturbed_identity_network(perturbation_magnitude) result(trainable_eng
     end associate
   end function
 
-end program
+end program

fpm.toml

@@ -1,10 +1,9 @@
 name = "inference-engine"
 version = "0.5.0"
 license = "license"
-author = "Damian Rouson, Tan Nguyen, Jordan Welsman, David Torres, Brad Richardson"
+author = "Damian Rouson, Tan Nguyen, Jordan Welsman, David Torres, Brad Richardson, Katherine Rasmussen"
 maintainer = "rouson@lbl.gov"
 
 [dependencies]
 assert = {git = "https://github.com/sourceryinstitute/assert", tag = "1.5.0"}
 sourcery = {git = "https://github.com/sourceryinstitute/sourcery", tag = "4.5.0"}
-netcdf-interfaces = {git = "https://github.com/LKedward/netcdf-interfaces.git", rev = "d2bbb71ac52b4e346b62572b1ca1620134481096"}

setup.sh

@@ -7,10 +7,10 @@ usage()
   echo "Inference Engine Setup Script"
   echo ""
   echo "USAGE:"
-  echo "./setup.sh [--help|-h] | [-p|--prefix=PREFIX]"
+  echo "./setup.sh [--help|-h]"
   echo ""
   echo " --help             Display this help text"
-  echo " --prefix=PREFIX    Install binary in 'PREFIX/bin'"
+  echo " --prefix=PREFIX    Install any binaries needed to build inference-engine in 'PREFIX/bin'"
   echo "                    Default prefix='\$HOME/.local/bin'"
   echo ""
 }
@@ -39,100 +39,53 @@ done
 
 set -u # error on use of undefined variable
 
-if ! command -v brew > /dev/null ; then
+install_fpm_from_source()
+{
+  echo "Installing fpm in following location: $PREFIX/bin"
+  echo "Ensure $PREFIX/bin is in your path or rerun script with the --prefix=PREFIX flag"
   if ! command -v curl > /dev/null ; then
+    echo "This script uses curl to download source file for fpm to install it"
     echo "Please install curl and then rerun ./setup.sh"
     exit 1
   fi
-  /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-  if [ $(uname) = "Linux" ]; then
-    if [ -z "$PATH" ]; then
-      PATH=/home/linuxbrew/.linuxbrew/bin/
-    else
-      PATH=/home/linuxbrew/.linuxbrew/bin/:"$PATH"
+  mkdir temp-dir-to-build-fpm-for-inference-engine-installation
+  curl -L -o temp-dir-to-build-fpm-for-inference-engine-installation/fpm.F90 https://github.com/fortran-lang/fpm/releases/download/current/fpm.F90
+  gfortran -o $PREFIX/bin/fpm -Jtemp-dir-to-build-fpm-for-inference-engine-installation temp-dir-to-build-fpm-for-inference-engine-installation/fpm.F90
+  rm -rf temp-dir-to-build-fpm-for-inference-engine-installation
+  if command -v fpm > /dev/null ; then
+    echo "fpm installed"
+  else
+    echo "Some error has occured while trying to install fpm. Please install fpm, ensure it is in your path, and rerun script"
+  fi
+}
+
+# if no fpm, install either through homebrew, or gfortran compiling fpm.F90
+if ! command -v fpm > /dev/null ; then
+  if ! command -v brew > /dev/null ; then
+    if ! command -v gfortran > /dev/null ; then
+      echo "Please install fpm, ensure it is in your path, and rerun script"
+      exit 1
+    else # has gfortran, but not homebrew
+      install_fpm_from_source
     fi
+  else # has homebrew
+    brew tap fortran-lang/fortran
+    brew install fortran-lang/fortran/fpm
   fi
 fi
 
-
-brew tap fortran-lang/fortran # required for building fpm
-brew install fpm netcdf netcdf-fortran pkg-config coreutils # coreutils supports `realpath` below
-
-PREFIX=`realpath $PREFIX`
-
-NETCDF_LIB_PATH="`brew --prefix netcdf`/lib"
-HDF5_LIB_PATH="`brew --prefix hdf5`/lib"
-NETCDFF_LIB_PATH="`brew --prefix netcdf-fortran`/lib"
-
-FPM_LD_FLAG=" -L$NETCDF_LIB_PATH -L$HDF5_LIB_PATH -L$NETCDFF_LIB_PATH"
-FPM_FLAG="-fcoarray=single -O3 -fallow-argument-mismatch -ffree-line-length-none -L$NETCDF_LIB_PATH -L$HDF5_LIB_PATH"
 FPM_FC=${FC:-"gfortran-13"}
 FPM_CC=${CC:-"gcc-13"}
 
 mkdir -p build
 
-CI=${CI:-"false"} # GitHub Actions workflows set CI=true
-
-if [ $CI = true ]; then
-  PKG_CONFIG_PATH=`realpath ./build/pkgconfig`
-  echo "---------------"
-  echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH"
-  echo "---------------"
-else
-  PKG_CONFIG_PATH=`realpath "$PREFIX"/lib/pkgconfig`
-fi
-
-if [ ! -d $PKG_CONFIG_PATH ]; then
-  mkdir -p $PKG_CONFIG_PATH 
-fi
-
-INFERENCE_ENGINE_PC="$PKG_CONFIG_PATH/inference-engine.pc"
-echo "INFERENCE_ENGINE_FPM_CC=\"$FPM_CC\""                  >  $INFERENCE_ENGINE_PC
-echo "INFERENCE_ENGINE_FPM_FC=\"$FPM_FC\""                  >> $INFERENCE_ENGINE_PC
-echo "INFERENCE_ENGINE_FPM_LD_FLAG=\"$FPM_LD_FLAG\""        >> $INFERENCE_ENGINE_PC
-echo "INFERENCE_ENGINE_FPM_FLAG=\"$FPM_FLAG\""              >> $INFERENCE_ENGINE_PC
-echo "Name: inference-engine"                               >> $INFERENCE_ENGINE_PC
-echo "Description: Inference Engine"                        >> $INFERENCE_ENGINE_PC
-echo "URL: https://github.com/berkeleylab/inference-engine" >> $INFERENCE_ENGINE_PC
-echo "Version: 0.1.2"                                       >> $INFERENCE_ENGINE_PC
-if [ $CI = true ]; then
-  echo "---------------"
-  echo "cat $INFERENCE_ENGINE_PC"
-  cat $INFERENCE_ENGINE_PC
-  echo "---------------"
-fi
-
-export PKG_CONFIG_PATH
-cp scripts/run-fpm.sh-header build/run-fpm.sh
-RUN_FPM_SH="`realpath ./build/run-fpm.sh`"
-echo "`which fpm` \$fpm_arguments \\" >>  $RUN_FPM_SH
-echo "--profile release \\" >> $RUN_FPM_SH
-echo "--c-compiler \"`pkg-config inference-engine --variable=INFERENCE_ENGINE_FPM_CC`\" \\" >> $RUN_FPM_SH
-echo "--compiler \"`pkg-config inference-engine --variable=INFERENCE_ENGINE_FPM_FC`\" \\" >> $RUN_FPM_SH
-echo "--flag \"-cpp `pkg-config inference-engine --variable=INFERENCE_ENGINE_FPM_FLAG`\" \\"  >> $RUN_FPM_SH
-echo "--link-flag \"`pkg-config inference-engine --variable=INFERENCE_ENGINE_FPM_LD_FLAG`\" \\" >> $RUN_FPM_SH
-echo "\$program_arguments" >> $RUN_FPM_SH
-chmod u+x $RUN_FPM_SH
-if [ $CI = true ]; then
-  echo "---------------"
-  echo "cat $RUN_FPM_SH"
-  cat $RUN_FPM_SH
-  echo "---------------"
-fi
-
-if command -v fpm > /dev/null 2>&1; then
-  brew tap fortran-lang/fortran
-  brew install fpm
-fi
-
-echo "$RUN_FPM_SH test"
-$RUN_FPM_SH test
+fpm test
 
 echo ""
-echo "____________________ Inference-Engine has been set up! _______________________" 
+echo "____________________ Inference-Engine has been set up! _______________________"
 echo ""
 echo "To run one of the programs in the example subdirectory, enter a command of the"
 echo "following form at a shell command prompt after replacing <example-base-name>"
 echo "with the base name of a file in the example/ subdirectory:"
 echo ""
-echo "./build/run-fpm.sh run --example <example-base-name>"
+echo "fpm run --example <example-base-name> --profile release"