-- DavidOvens - 18 Mar 2008

good documentation also at:
http://www.dtcenter.org/wrf-nmm/users/docs/presentations/index.php
 

# installing WRF 2.2 on oldsage Jan 2007 (see also ~ovens/notes/wps.notes):
  cd wrf2007
  set MAINDIR = $PWD

  setenv NETCDF /usr/local/netcdf-ifort
  if (`uname -m` == "x86_64") setenv NETCDF /usr
  setenv NCARG_ROOT /usr/local/ncarg-ifort

  setenv WRF_EM_CORE 1
  source /usr/local/gempak/Gemenviron
  echo $GEMEXE | grep 5.8
  if ($status == 0) then
# GEMPAK 5.9 and 5.8 settings differ
# these are 5.8 settings
   setenv JASPERLIB $NAWIPS/jasper/linux/lib
   setenv JASPERINC $NAWIPS/jasper/linux/include
   setenv PNGLIB $NAWIPS/lib/linux
   setenv ZLIB $NAWIPS/lib/linux
   setenv ZLIBINC $NAWIPS/gempak/include
  else
# these are 5.9 settings
   setenv JASPERLIB $GEMLIB
   setenv JASPERINC $NAWIPS/extlibs/JasPer/v1.701.0/src/libjasper/include
   setenv PNGLIB $GEMLIB
   setenv ZLIB $GEMLIB
  ## not sure about this next one
   setenv ZLIBINC $NAWIPS/gempak/include
  endif

##
## one time only:
  tar zxvf geog.tar.gz

##
## for each new WPS and WRFV2
  tar zxvf ~/code_repository/WPSV2.2.TAR.gz
  tar zxvf ~/code_repository/WRFV2.2.TAR.gz

## 
## for each new domain
set GEOG = /home/disk/sage2/mm5rt/wps/geog
set DOMAIN = $MAINDIR/domains/may06
mkdir -p $DOMAIN

##
## compile WRF first
cd WRFV2
ifort -v
if (`uname -m` == "x86_64") then
## use option 9 on tahoma/rainier x86_64 machines 
## (option 7 does not allow nesting,
##  option 8 allows nesting, and seems to want to try to do multiple
##  threads, but it really only uses 1.4 CPU... lame)
  ./configure <<EOF
9
EOF  
else
## oldsage cluster
  ./configure <<EOF
12
EOF  
endif

emacs configure.wrf &
## KEY MISSING ITEM: -P in
## CPP             =       /lib/cpp -traditional
## CPP             =       /lib/cpp -P -traditional

## ALSO ADD:
## LINUX_MPIHOME   =      /usr/local/mpich-ifort-1.2.5.2

## ALSO CHANGE TO:
## FC              =       $(LINUX_MPIHOME)/bin/mpif90 -f90=ifort
## LD              =       $(LINUX_MPIHOME)/bin/mpif90 -f90=ifort
## CC              =       $(LINUX_MPIHOME)/bin/mpicc -cc=gcc -DMPI2_SUPPORT -DFSEEKO64_OK 
## FCOPTIM      =   -O2 -xW -tpp7
## FCBASEOPTS      =       -FR -cm -w -I. $(FCDEBUG) -convert big_endian
## KEY MISSING ITEM: -P in
## CPP             =       /lib/cpp -traditional
## CPP             =       /lib/cpp -P -traditional

## ALSO ADD:
## LINUX_MPIHOME   =      /usr/local/mpich-ifort-1.2.5.2

##
## did not make the wrfio_grib2 changes that I had to do with the
## pre-release.  If problems appear due to GRIB2, bacio, g2lib, see
## the ~ovens/notes/wps.notes file for info on what I had to do with
## the pre-release.

##
## compile the code
  ./compile em_real | & tee make.out
##  --> took 3 hours!!!!!!!!!!!!!!!!
##  --> took 3 hours!!!!!!!!!!!!!!!!

##
## now compile WPS
cd ../WPS
if (! -e arch/preamble.orig) then
  cp -p arch/preamble arch/preamble.orig
  emacs arch/preamble
##
## I add LINUX_MPIHOME stuff right before SUFFIXES line:
LINUX_MPIHOME   =      /usr/local/mpich-ifort-1.2.5.2
## new tahoma one is this:
LINUX_MPIHOME   =      /usr/local/mpich-ifort-2.1.0.4p1

##
## MUST change hard-coded:
COMPRESSION_LIBS   =   -L/data3a/mp/gill/WPS_LIBS/local/lib \
            -ljasper -lpng12 -lpng -lz

COMPRESSION_INC      =   -I/data3a/mp/gill/WPS_LIBS/local/include

## to
COMPRESSION_LIBS   =   -L$(JASPERLIB) -L$(PNGLIB) \
            -ljasper -lpng12 -lpng -lz

COMPRESSION_INC      =   -I$(JASPERINC)

endif


if (! -e arch/configure.defaults.orig) then
  cp -p arch/configure.defaults arch/configure.defaults.orig
  emacs arch/configure.defaults
## on oldsage, look for and edit the
#ARCH   PC Linux i486 i586 i686, Intel compiler   DM parallel
# section as follows:
##
## I also do add LINUX_MPIHOME stuff:
FC      =   mpif90 -f90=ifort
## -->
FC      =   $(LINUX_MPIHOME)/bin/mpif90 -f90=ifort

## and
CC      =   mpicc -cc=icc
## -->
CC      =   $(LINUX_MPIHOME)/bin/mpicc -cc=gcc

## and I add NCARG stuff for using pgf90 instead of ifort
NCARGFC      =   pgf90
NCARGLDFLAGS   =   -byteswapio


## on tahoma, rainier, look for and edit the
#ARCH    PC Linux x86_64 (IA64 and Opteron), PGI compiler 5.2 or higher, serial
# section as follows:
##
## change
COMPRESSION_LIBS   =   -L/contrib/zlib/lib -lz \
            -L/contrib/jasper/lib -ljasper \
            -L/contrib/libpng/lib -lpng
COMPRESSION_INC      =   -I/contrib/zlib/include \
            -I/contrib/jasper/include \
            -I/contrib/libpng/include
FC      =   pgf90
SFC      =   pgf90
FFLAGS      =   -Mfree -byteswapio -O
F77FLAGS   =   -byteswapio -O
FNGFLAGS   =   $(FFLAGS)
LDFLAGS      =   
CC      =   pgcc
SCC      =   pgcc
CFLAGS      =   -O
CPP      =   /lib/cpp -C -P -traditional
CPPFLAGS   =   -D_UNDERSCORE -DBYTESWAP -DLINUX -DIO_NETCDF -DIO_BINARY -DIO_GRIB1 -DBIT32
NCARG_LIBS2   =   -L${PGI}/linux86-64/5.2/lib -lpgftnrtl -lpgc \
         -L/usr/lib64 -lg2c

##
## to
FC      =   $(LINUX_MPIHOME)/bin/mpif90 -f90=ifort
SFC      =   ifort
FFLAGS      =   -FR -convert big_endian
F77FLAGS   =   -convert big_endian
FNGFLAGS   =   $(FFLAGS)
LDFLAGS      =   
CC      =   $(LINUX_MPIHOME)/bin/mpicc -cc=gcc
SCC      =   gcc
CFLAGS      =   -w
CPP      =   /lib/cpp -C -P -traditional
CPPFLAGS   =   -D_UNDERSCORE -DBYTESWAP -DLINUX -DIO_NETCDF -DIO_BINARY -DIO_GRIB1 -D_MPI -DBIT32


endif

if (`uname -m` == "x86_64") then
## use option 2 on tahoma/rainier x86_64 machines
./configure <<EOF
2
EOF
else
## use option 8 on oldsage cluster
./configure <<EOF
8
EOF
endif

if (! -e geogrid/src/parallel_module.F90.bad) then
   cp -p geogrid/src/parallel_module.F90 geogrid/src/parallel_module.F90.bad
   emacs geogrid/src/parallel_module.F90

## MUST delete MPI_Bcast calls for MPI_CHARACTER in
## geogrid/src/parallel_module.F90 with this:
      if (present(from_whom)) then
!         call MPI_Bcast(cval, n, MPI_CHARACTER, from_whom, comm, mpi_ierr)
      else
!         call MPI_Bcast(cval, n, MPI_CHARACTER, IO_NODE, comm, mpi_ierr)
      end if
      print *,'cannot run mpi_bcast on char...'
      call exit(1)

endif

if (! -e ungrib/src/Makefile.orig) then
  cp -p ungrib/src/Makefile ungrib/src/Makefile.orig
  emacs ungrib/src/Makefile
##
## MUST CHANGE:
##
g2print.exe:      gridinfo.o g2print.o
         $(FC) -o $(@) $(LDFLAGS) g2print.o \
## to
g2print.exe:      gridinfo.o g2print.o filelist.o gridinfo.o
         $(FC) -o $(@) $(LDFLAGS) g2print.o filelist.o gridinfo.o \

endif

./compile |& tee make.out


pushd $DOMAIN
emacs $MAINDIR/WPS/namelist.wps &
cp $MAINDIR/WPS/namelist.wps .

set WPSDIR = $MAINDIR/WPS
## 
## one time only

## NOTE that the opt_output_from_geogrid_path value setting of ./ will
## put the output into a path relative to the path to executable! 
## if the the executable is not the current directory, ./ will not be
## the current directory, but the directory where the executable is
## located.
ln -s $WPSDIR/geogrid.exe .
./geogrid.exe |& tee geogrid.out

setenv RIP_ROOT $ms/rt/postprocess/RIP4


########################################################################
## Run ungrib
## edit share and ungrib sections of namelist.wps file
set date = 2006090712
set ym = `echo $date | cut -c1-6`

mkdir $ms2/wrf/$date.wps3
cd $ms2/wrf/$date.wps3
cp $WPSDIR/namelist.wps .
emacs namelist.wps&
## after setting start and end dates and confirming output format,
## we must supply a Vtable and the GRIB files must be linked to
## the filenames that are expected by ungrib.  Many Vtables are
## already supplied and can just be linked in as 'Vtable'.
## e.g. ln -s ungrib/Vtable.GFS Vtable
##
## Wants files named GRIBFILE.AAA..., so use link_grib.csh to do this:
cp  $WPSDIR/ungrib/Variable_Tables/Vtable.GFS Vtable


## past case
  mkdir input.grids
  cd input.grids
  cp $ms/data/input.grids/$ym/$date/*gfs* .
  gunzip *gz; bunzip2 *bz2
  cd ..
  set inputdir = $PWD/input.grids

## normal case
  set inputdir = $ms/data/input.grids

$WPSDIR/link_grib.csh $inputdir/$date*gfs.grb
$WPSDIR/ungrib.exe 
ls -l FIL*
rm GRIBFILE*


#In the final step of running WPS, the meteorological data extracted
# by ungrib are horizontally interpolated to the simulation grids
# defined by geogrid. In order to run metgrid, the namelist.wps file
# must be edited. In particular, the share and
# metgrid namelist sections are of relevance to the metgrid
# program. Examples of these sections are shown below. 

# note that for metgrid, the opt_output_from_geogrid_path value setting
# of ./ will look for the output from geogrid in current directory
# --> needed to change that to a full path

# Also, metgrid needed to have this set:
#  opt_metgrid_tbl_path         = '/home/disk/sage2/mm5rt/wps/WPS/metgrid/',

# Also, if 
#  opt_output_from_metgrid_path = './',
# is set -- the default -- and you run the command as $path/metgrid.exe,
# the output does not go to ./ but to $path.


## Then created namelist.input and ran real.exe and wrf.exe off of that.

ln -s $WPSDIR/metgrid.exe .
./metgrid.exe  |& tee metgrid.out

## run real.exe
#create a namelist.input file

cp $MAINDIR/WRFV2/run/namelist.input .
emacs namelist.input &
## set up the fdda stuff here, before real.exe
##
 &fdda
 grid_fdda                           = 1, 0,
 gfdda_inname                        = "wrffdda_d<domain>"
 gfdda_interval_m                    = 180, 0,
 gfdda_end_h                         = 180, 0,
 io_form_gfdda                       = 2
 fgdt                                = 0, 0,
 if_no_pbl_nudging_uv                = 0
 if_no_pbl_nudging_t                 = 1
 if_no_pbl_nudging_q                 = 1
 if_zfac_uv                          = 1, 0
  k_zfac_uv                          = 10, 10
 if_zfac_t                           = 0, 0
  k_zfac_t                           = 10, 10
 if_zfac_q                           = 0, 0
  k_zfac_q                           = 10, 10
 guv                                 = 0.0001, 0.0001
 gt                                  = 0.0001, 0.0001
 gq                                  = 0.000001
 if_ramping                          = 0
 dtramp_min                          = 0.0
 /

if (`uname -m` == "x86_64") then
  /usr/local/mpich-ifort-2.1.0.4p1/bin/mpiexec -np 4 $MAINDIR/WRFV2/main/real.exe
else
  /usr/local/mpich-ifort-1.2.5.2/bin/mpirun -np 1 $MAINDIR/WRFV2/main/real.exe
endif


ncdump -v Times wrfbdy*
ncdump -v Times wrfinput_d01
ncdump -v Times wrffdda_d01

cp $MAINDIR/WRFV2/run/[C-V]* .
cp $MAINDIR/WRFV2/run/{grib,ozone,tr,urban}* .
cp $MAINDIR/WRFV2/run/wrf.exe .

if (`uname -m` == "x86_64") then
  time /usr/local/mpich-ifort-2.1.0.4p1/bin/mpiexec -np 4 $MAINDIR/WRFV2/main/wrf.exe
else 
 /usr/local/mpich-ifort-1.2.5.2/bin/mpirun -np 1 $PWD/wrf.exe
endif
========================================================================

installing WRF 2.0.2 on sage:
references:
 http://www.mmm.ucar.edu/wrf/users/docs/user_guide/users_guide_chap2.html#Introduction
   cd $ms3/wrf_tests
   gunzip -c ../WRFV2.0.2.TAR.gz | tar xvf -
   setenv NETCDF /usr/local
   setenv WRF_EM_CORE 1
   limit stacksize 480M
   setenv MP_STACK_SIZE 64000000
   setenv MPICH_F90 pgf90
   setenv OMP_NUM_THREADS 1 # for now...
   
   cd WRFV2/
   ##
   ## Note, the single-processor wrf.exe which I compiled first
   ## does not work with nests.  Need to use my configure.wrf.mpi.
   ./configure
      1
   time ./compile em_real  ## this took 14:06
   ls -ls main/*.exe


   ## Now, attempt wrfsi
for 200603 build, see /home/disk/sage2/mm5rt/wrf2006/wrfsi/commands

references:
 http://www.mmm.ucar.edu/wrf/users/docs/user_guide/users_guide_chap2.html#Introduction
 http://www.mmm.ucar.edu/wrf/users/docs/user_guide/users_guide_chap3.html#Introduction
 wrfsi/INSTALL:
 http://www.wrf-model.org/si/wrfsiREADME.html
   gunzip -c ../../wrfsi_v2.0.1.tar.gz | tar xvf -
   cd wrfsi
   setenv NETCDF /usr/local
   setenv MPICH /usr/local/mpich-1.2.5.bad
   mkdir /home/disk/sage2/mm5rt/wrf_tests/domains
   mkdir /home/disk/sage2/mm5rt/wrf_tests/input.grids
   ##
   ## check for updates to global terrain, landuse, soil at
   ## ftp://aftp.fsl.noaa.gov/divisions/frd-laps/WRFSI/Geog_Data/
   ## and place into /home/disk/sage2/mm5rt/wrf_tests/terrain 
   perl -d install_wrfsi.pl \
    --source_root=/home/disk/sage2/mm5rt/wrf_tests/WRFV2/wrfsi \
    --installroot=/home/disk/sage2/mm5rt/wrf_tests/WRFV2/wrfsi \
    --dataroot=/home/disk/sage2/mm5rt/wrf_tests/domains \
    --ext_dataroot=/home/disk/sage2/mm5rt/wrf_tests/input.grids \
    --geog_dataroot=/home/disk/sage2/mm5rt/wrf_tests/terrain \
    --templates=/home/disk/sage2/mm5rt/wrf_tests/WRFV2/wrfsi/templates \
    --path_to_netcdf=/usr/local
   ## paths output to:
   ## $INSTALLROOT/config_paths
   ##
   ## Instructions then say to:
   ## 
   ## -- Set up grib_prep to ingest your GRIB data sets:
   ##  a.  cd /home/disk/sage2/mm5rt/wrf_tests/input.grids/static
   ##  b.  Edit grib_prep.nl and/or the Vtables
   ## -- See /home/disk/sage2/mm5rt/wrf_tests/WRFV2/wrfsi/HOW_TO_RUN.txt for more info
   setenv INSTALLROOT /home/disk/sage2/mm5rt/wrf_tests/WRFV2/wrfsi 
   setenv SOURCE_ROOT /home/disk/sage2/mm5rt/wrf_tests/WRFV2/wrfsi 
   setenv DATAROOT   /home/disk/sage2/mm5rt/wrf_tests/domains
   setenv EXT_DATAROOT /home/disk/sage2/mm5rt/wrf_tests/input.grids 
   setenv GEOG_DATAROOT /home/disk/sage2/mm5rt/wrf_tests/terrain 
   setenv TEMPLATES /home/disk/sage2/mm5rt/wrf_tests/WRFV2/wrfsi/templates 

   ## 
   ## Step 1: Localize the simulation domain and create static fields - window_domain_rt.pl
   ##setenv MOAD_DATAROOT is a new case directory
   setenv MOAD_DATAROOT /home/disk/sage2/mm5rt/wrf_tests/domains/dec03
   cd $INSTALLROOT/templates
   cp -r default $MOAD_DATAROOT:t
   chmod -R ug+w $MOAD_DATAROOT:t
   emacs $MOAD_DATAROOT:t/wrfsi.nl
    ## paid particular attention, as User's Guide Chap. 3 Step 1 (for
    ##  window_domain_rt.pl) tells me, to the &hgridspec section
   $INSTALLROOT/etc/window_domain_rt.pl -w wrfsi -c -t $TEMPLATES/$MOAD_DATAROOT:t
    ## this in turn, runs this:
    ## $INSTALLROOT/etc/localize_domain.pl --dataroot=/home/disk/sage2/mm5rt/wrf_tests/domains/dec03 --srcroot=/home/disk/sage2/mm5rt/wrf_tests/WRFV2/wrfsi --install_root=/home/disk/sage2/mm5rt/wrf_tests/WRFV2/wrfsi --which_type=wrfsi > /home/disk/sage2/mm5rt/wrf_tests/domains/dec03/log/localize_domain.log
  ## 
  ## this did not work at all with '-t $TEMPLATES'.  It created a
  ##  single domain over the midwest!  some stupid default, 144x144
  ##  grid.  They have now updated the User's Guide to the CRUCIAL
  ##  '-t $TEMPLATES/$MOAD_DATAROOT:t'.

   ## 
   ## Step 2: Debrib GRIB files - grib_prep.exe

   emacs $EXT_DATAROOT/static/grib_prep.nl ## edit &gpinput_defs section

! The grib_prep.nl namelist is used to control execution of grib_prep, the 
! program WRFSI uses to decode GRIB files into the binary intermediate format
! used by hinterp.  It is composed of two namelist sections, "filetimespec"
! and "gpinputdefs".  The grib_prep.exe executable expects to find this
! file in the working directory from which it is executed, along with
! a series of GRIB files to process named "GRIBFILE.AA", "GRIBFILE.AB", 
! etc.  The grib_prep.exe program only uses the filetimespec section of the
! namelist.  The gpinputdefs section is used by the perl script "grib_prep.pl"
! to set up the EXT_DATA_ROOT/work/SOURCE subdirectory from which to run
! the executable.  If you are not using grib_prep.pl to run grib_prep.exe, then
! you will need to manually set up everything that the script takes care of.
   ##
   ## Since I have gone to so much trouble to create pregrid.pl, I am going
   ##  to skip their grib_prep.pl and use my own in the future.
   perl -d $INSTALLROOT/etc/grib_prep.pl -s 2004092812 -l 72 \
-t 3 -f '^2004092812_0[0-7]\d_gfs.grb' GFS
   perl $INSTALLROOT/etc/grib_prep.pl -s 2004092712 -l 0 \
-t 24 -f '^2004092\d12.sea_temp' OTISSST
   perl $INSTALLROOT/etc/grib_prep.pl -s 2004092812 -l 0 \
-t 24 -f '^20040928\d\d_221.t.grb' ETA221
   perl $INSTALLROOT/etc/grib_prep.pl -s 2004092812 -l 0 \
-t 24 -f '^20040928\d\d_221.t.grb' ETA221SOIL
   perl $INSTALLROOT/etc/grib_prep.pl -s 2004092812 -l 0 \
-t 24 -f '^20040928(00|12)_000_ruc20soil.grb' RUC20

   ##trial
   cd $EXT_DATAROOT/extprd
   cat ETA221SOIL:2004-09-28_12 OTISSST:2004-09-27_12 > ETA221.SNOW_SOIL_OTIS.SST_FILE:2004-09-28_12

   ##
   ## NOTE: Vtables are slightly different than those used in MM5's
   ##       pregrid -- the soil parameter output names have changed
   ##
   ## NOTE: $INSTALLROOT/src/grib_prep/grib.misc/grib_prep.F is
   ##       $MM5/rt/preprocess/REGRID/pregrid/grib.misc/pregrid_grib.F
   ## differences are in output.F, add map%source, map%startloc
   ##                    rd_grib.F -- will want to modify/update
   ##                                 to handle UKMO, thinned grids, etc.

   ## 
   ## Step 3: Interpolating meteorological data - wrfprep.pl
   ## runs hinterp and vinterp
   ##
   ## make MOAD_DATAROOT/static/wrfsi.nl consistent with
   ##   $TEMPLATES/$MOAD_DATAROOT:t/wrfsi.nl
   ## pay attention here to &interp_control section
   emacs $MOAD_DATAROOT/static/wrfsi.nl
   ##
   perl -d $INSTALLROOT/etc/wrfprep.pl -s 2004092812 -f 72

 --> trying:
    cp $MOAD_DATAROOT/static/wrfsi.nl.0hour $MOAD_DATAROOT/static/wrfsi.nl
    perl -d $INSTALLROOT/etc/wrfprep.pl -s 2004092812 -f 0
     b 926
     system('pwd') tells us running in $MOAD_DATAROOT/siprd
    hinterp fails for domain 2 with a segmentation fault on the SST
    field... but this does work for GFS 
  --> trying:
      perl $INSTALLROOT/etc/grib_prep.pl -s 2004092812 -l 0 \
-t 24 -f '^20040928\d\d_221.t.grb' ETA221FULL
    cp $MOAD_DATAROOT/static/wrfsi.nl.0hour.eta221full $MOAD_DATAROOT/static/wrfsi.nl
    perl -d $INSTALLROOT/etc/wrfprep.pl -s 2004092812 -f 0
## THIS WORKED !!!!!
## THIS WORKED !!!!!
  --> trying:
      perl $INSTALLROOT/etc/grib_prep.pl -s 2004092812 -l 0 \
-t 24 -f '^20040928\d\d_221.t.grb' ETA221LSM
      cp $MOAD_DATAROOT/static/wrfsi.nl.allhours.gfs.eta221lsm $MOAD_DATAROOT/static/wrfsi.nl
##      cp $MOAD_DATAROOT/static/wrfsi.nl.0hour.gfs.eta221lsm $MOAD_DATAROOT/static/wrfsi.nl
      cd $MOAD_DATAROOT/siprd
      ln -s $EXT_DATAROOT/extprd/{GFS,OTISSST,ETA221LSM}* .
      $INSTALLROOT/bin/hinterp.exe |& tee WRFHELP.hinterp.out
      foreach i (hinterp*) 
         cp $i WRFHELP.$i
      end
      $INSTALLROOT/bin/vinterp.exe |& tee WRFHELP.vinterp.out
      foreach i (wrf_real*) 
         cp $i WRFHELP.$i
      end

11/23
I am wondering, is my meshing just putting temperature into T when it
is really supposed to be potential temperature?  A look at a single,
standard run should be able to confirm that it's potential
temperature.  A my siprd.eta221otis/wrf_real* files have T not THETA,
but siprd.mesh2/wrf_real* have THETA.  Which is correct?

11/23 tried:
  ::  cp $MOAD_DATAROOT/static/wrfsi.nl.fulltimerange.mixedconstants $MOAD_DATAROOT/static/wrfsi.nl
    mkdir $MOAD_DATAROOT/siprd.mixedconstants
    cd $MOAD_DATAROOT/siprd.mixedconstants
    ln -s $EXT_DATAROOT/extprd/{GFS,ETA221.SNOW}* .
    $INSTALLROOT/bin/hinterp.exe |& tee MIXEDCONST.hinterp.out
  AGAIN, THIS CRASHED WITH SEGMENTATION FAULT.  The mixed constants
    approach does not work!

  ::   now, try spoofing ETA221LSM files (internal dates will be the start
    time, but file name dates will look like forecast times):    

    cp $MOAD_DATAROOT/static/wrfsi.nl.allhours.gfs.eta221lsm $MOAD_DATAROOT/static/wrfsi.nl
    mkdir siprd.multihourlsm
    cd siprd.multihourlsm
    ln -s $EXT_DATAROOT/extprd/{GFS,OTISSST,ETA221LSM}* .
    set date = 2004092812
    while ($date <= 2004100112)
      set date = `time-increment "${date}0000" yy 0 10800 YYYYMMDDHH`
      set fdate = `time-increment "${date}0000" yy 0 0 "YYYY-MM-DD_HH"`
      echo "ln -s ETA221LSM:2004-09-28_12 ETA221LSM:$fdate"
      ln -s ETA221LSM:2004-09-28_12 ETA221LSM:$fdate
    end
    $INSTALLROOT/bin/hinterp.exe |& tee MULTIHOURLSM.hinterp.out
 ==> FAILED, spoofed external date didn't fool it.  The internal date
    overruled. 

 Now, I think I should try testing a quick modification to MM5 PREGRID
 to output into wrfsi format.  Then, I can go back to the previous
 test with the constants file and see if that works.  Hmm...or should
 I just use my mesh code on all GFS:FILE* files?  result should be the
 same...

 :: 11/24 after talking with Rick, I think that the LSM approach is
 actually very close to doing what I want.  He also has used multiple
 files in the CONSTANTS in MM5, so this may work for WRF.
  . created Vtable.GFS2, Vtable.ETA221LSM2, Vtable.ETA221CONSTANT in
    $EXT_DATAROOT/static/ 
  . edited $EXT_DATAROOT/static/grib_prep.pl
  . ran grib prep
   perl $INSTALLROOT/etc/grib_prep.pl -s 2004092812 -l 0 -t 24 -f \
 '^20040928\d\d_221.t.grb' ETA221CONSTANT 
   perl $INSTALLROOT/etc/grib_prep.pl -s 2004092812 -l 0 -t 24 -f \
 '^20040928\d\d_221.t.grb' ETA221LSM2
   perl -d $INSTALLROOT/etc/grib_prep.pl -s 2004092812 -l 72 \
-t 3 -f '^2004092812_0[0-7]\d_gfs.grb' GFS
  . now go for it..
   cp $MOAD_DATAROOT/static/wrfsi.nl.allhours.gfs2.lsm2.constant $MOAD_DATAROOT/static/wrfsi.nl
    mkdir $MOAD_DATAROOT/siprd.gfs2lsm2constant
    cd $MOAD_DATAROOT/siprd.gfs2lsm2constant
    ln -s $EXT_DATAROOT/extprd/{GFS2,ETA221LSM2,ETA221CONST,OTIS}* .
    $INSTALLROOT/bin/hinterp.exe |& tee GFS2LSM2CONSTANT.hinterp.out
   # worked !!
    $INSTALLROOT/bin/vinterp.exe |& tee GFS2LSM2CONSTANT.vinterp.out
   --- this worked, but output went into ../siprd not PWD !!!!
   --- this worked, but output went into ../siprd not PWD !!!!
   --- this worked, but output went into ../siprd not PWD !!!!
   --- this worked, but output went into ../siprd not PWD !!!!
   Answers were identical to WRFHELP because of the siprd problem!!!!  
   mv siprd siprd.wrfhelp
   mv siprd.gfs2lsm2constant siprd
   cd siprd
   $INSTALLROOT/bin/vinterp.exe |& tee GFS2LSM2CONSTANT.vinterp.out
   cd ..
   mv siprd siprd.gfs2lsm2constant
   mv siprd.wrfhelp siprd
 this makes me think that SPECHUMD and SOILHGT need to be dropped via
 further refinement of those Vtables... OR, I may just want to see if
 the wrfinput and wrfby files actually differ between this and the
 WRFHELP version.

   ##trial
   cd $EXT_DATAROOT/extprd
   cat ETA221SOIL:2004-09-28_12 OTISSST:2004-09-27_12 > ETA221.SNOW_SOIL_OTIS.SST_FILE:2004-09-28_12
   ## --> fails vinterp
    CONSTANTS_FULL_NAME = 'OTISSST:2004-09-27_12',
   works, but anything with ETA221SOIL stuff fails....

   ## 
   ## hinterp notes from User's Guide:
   ## 10. CONSTANTS_FULL_NAME: Specifies a list of file names to look
   ##     for in "CONSTANTS_PATH". Data contained in any of these
   ##     files actually found will be used at every output time, and
   ##     will take precedence over duplicate data found in
   ##     LSM_ROOT/INIT_ROOT/LBC_ROOT files.



  ##
  ## check output:
  $INSTALLROOT/bin/siscan $MOAD_DATAROOT/siprd/hinterp.d02.2004-09-28_12:00:00

  ## seems to have worked..., $MOAD_DATAROOT/siprd files hinterp*
  ## wrf_real_input_em* all exist

  setenv NCARG_ROOT /usr/local/ncarg
  setenv NCL_COMMAND /usr/local/ncarg/bin/ncl
  cd $INSTALLROOT/graphics/ncl
  ./generate_images.pl  ## puts images in $MOAD_DATAROOT/static/*cgm


  ##
  ## Preparing nested wrf input files using real.exe:
  I created $MOAD_DATAROOT/setenv.csh
  cd $INSTALLROOT/../test/em_real
  ln -s $MOAD_DATAROOT/siprd/wrf_real_input_em.d02* .
  rename . "wrf_real_input_em.d02*" d02 d01
  emacs namelist.input
  ## only need one time and only first column matters
  ./real.exe
  /bin/cp namelist.input namelist.d02.input
  /bin/cp namelist.d01.input namelist.input
  ./real.exe
  

  ##
  ## I recompiled wrf.exe with -tp p7 instead of -tp p6 (the default)
  ## and I still got the error.
  ./wrf.exe
   --> leads to a 'Floating Exception' really early on -- at the first
  ## time step -- so I need to examine the input data.  What about
  ##TSK??  I saw problems in real.exe about that didn't I?
  

########################################################################

Ryan's scripts:
  /home/disk/ridge/torn/wrf_scripts
Ryan's real-time:
  /home/disk/ridge/torn/real_time_dom

Terrain info:
 . as with 8/29/2003, I investigated the aug02/terrain.matchwrf
   files. The WRF terrain values, found in
   /home/sage2/mm5rt/wrf_tests/dataroot/mm5d1aug02/static are at MM5
   cross points with one extra in the i and j locations.  Hence, to
   see the match up, look at the second to last entries in WRF (MM5
   has **** in the last entries for the cross points, but WRF gives
   real numbers).
 . generated TERRAIN for MM5 to be used in WRF comparisons
   -- aug02/terrain.matchwrf/TERRAIN_DOMAIN2 is the file that matches 
    /home/disk/wrf1/mm5rt/wrf_tests/dataroot/mm5d1aug02/static/static.wrfsi 
    as best as I can do it.  This was generated in /ensm/TERRAIN on
    wrf18 by modifying src/setup.F changing A/6370./ to A/6371.2/.


some more links from training:
  http://box.mmm.ucar.edu/mm5/mpp/wrfbrowser
  http://www.mmm.ucar.edu/wrf/users/WRF_arch_03.doc

trying to install new wrfsi
  make WRF code
  cd WRFV1.3/wrfsi-1.3.2/
  setenv NETCDF /usr/local
  setenv GEOG_DATAROOT /home/disk/sage2/mm5rt/wrf_tests/terrain
  unsetenv EXT_DATAROOT
  perl install_wrfsi.pl


Setting up a new domain:
  setenv INSTALLROOT /home/disk/wrf1/mm5rt/wrf_tests/WRFV1/wrfsi
  setenv MOAD_DATAROOT /home/disk/wrf1/mm5rt/wrf_tests/dataroot/torn
  cd $INSTALLROOT/templates
  cp -r default $MOAD_DATAROOT:t
  chmod -R u+w $MOAD_DATAROOT:t
  emacs $MOAD_DATAROOT:t/wrfsi.nl
  cd $INSTALLROOT/etc
  ./window_domain_rt.pl -w wrfsi -t \
$INSTALLROOT/templates/$MOAD_DATAROOT:t -c -s $INSTALLROOT
  cd $INSTALLROOT/etc
  ./wrfsi.pl -h
  SRCPATH in
   /home/disk/wrf1/mm5rt/wrf_tests/extdataroot/static/grib_prep.nl
   shows where programs will look for input data
  pushd /home/disk/wrf1/mm5rt/wrf_tests/dataroot/mm5d1aug02/gfs
  ln -s /home/disk/mm5rt/data/input.grids/2003081200_*avn.grb .
  cd $INSTALLROOT/etc
  ./wrfsi.pl 2003081200 24 GFS

grib_prep.pl (now run by wrfsi.pl) ==== PREGRID output format is
    slightly different, files are bigger
-rw-rw-r--    1 mm5rt    rmm5rt   35210160 Sep  3 22:10 GFS.FILE:2003-08-29_12
-rw-rw-rw-    1 mm5rt    rmm5rt   37824120 Sep  3 22:02 GFS:2003-08-29_12

wrfprep.pl   (now run by wrfsi.pl) ==== REGRIDDER

problems:
 x) what is the difference between
     WRFV1/wrfsi/templates/mm5d1aug02/wrfsi.nl
       ($INSTALLROOT/templates/mm5d1aug02/wrfsi.nl)
    and
     dataroot/mm5d1aug02/static/wrfsi.nl 
       ($MOAD_DATAROOT/static/wrfsi.nl) ?
 1) README.namelist in test/em_real/ (../../run/README.namelist) needs
 updating.  Specifically, io_form_initial is now io_form_input.
 2) wrfsi.nl in $INSTALLROOT/templates/default needs to be updated
 there are very large portions which are missing

 x) All logging should be to STDOUT by default, a user should specify
    "-log logfile" on the command line to redirect STDOUT and STDERR
    to a logfile.  Here's how I do it:
    while ($ARGV[0] ne "") {
      if .. {
      } elsif($ARGV[0] =~ /^-log/) {
      $logfile = $ARGV[1];
     shift; shift;
      }
...
    if ($logfile) {
   close(STDOUT);
   close(STDERR);
   open (STDOUT,">>$logfile");
   open(STDERR, ">&STDOUT");
    }

 x) where are the grib_prep filters specified? Mine always return
    nothing.
 
 x) in wrfprep.pl the workdir is $MOAD_DATAROOT/siprd and it cleans
    this directory -- obviously you cannot have 2 of these running at
    a time!!!  Working directory MUST BE cwd! 

 x) why do GFS:* files show up in EXT_DATAROOT/extprd?  shouldn't that
    be an external data directory?  Output should go to cwd.    
 

 x) in $MOAD_DATAROOT/static/wrfsi.nl the ANALPATH, LBCPATH, LSMPATH,
    CONSTANTS_PATH environment variables are used to point to paths
    for the input GRIB data -- in my case they are pointing to
    EXT_DATAROOT/extprd.  At no point do you specify that the Vtables
    are in EXT_DATAROOT/static or that you will be running in
    EXT_DATAROOT/work or that logging will go into EXT_DATAROOT/log.
    You are using a hard-coded directory structure rather than being
    explicit.  For my part, I want EXT_DATAROOT/work to be local to
    whatever machine I am running WRFSI on -- this speeds up program
    execution, whereas the GRIB files in EXT_DATAROOT/extprd are links
    to a big remote data disk -- definitely a place I do not wish to
    run on or attempt to write to.  You must be explicit about ALL
    paths that you will be using and you really should get rid of most
    of these directories anyways.  What you really need in terms of
    paths are:
     1) path to Vtables
     2) paths to GRIB data (SST, SOIL, MODEL, etc.)
     3) path to working directory
     4) path to output directory -- should default to current working
        directory unless specified differently on the command line.
    Also, do we really want to specify the starting and ending times
    of this particular forecast in this file AND in namelist.input AND
    to specify our initialization date on the command line?  Why not
    at least have wrfsi calculate this information based on the
    initialization date from the command line?.....<add more>...xs

    Also, major flaw in you "work" directory is that you cannot have 2
    wrfsi scripts running simultaneously -- they'll try to use the
    same directory!

 x) ARGH!!!! why are SRCPATHs and filetimespec specified in yet another
    namelist file -- EXT_DATAROOT/static/grib_prep.nl  why can't this
    be as used in wrfsi.nl????  -- Looks like maybe wrfprep.pl and
    grib_prep.pl might be somewhat intelligent about dates...

 x) why do I have to specify julday in namelist.input -- can't the
    code figure this out for me, especially if I have to re-specify
    all my run information in namelist_05.

 x) Why in blazes is MOAD_DATAROOT/ populated with subdirectories
    [cwd = current working directory]
     cdl     -- important information, get rid of this directory and
                put the contents into the new "static" directory
     log     -- get rid of this and output to cwd
     siprd   -- what does this stand for?  why would I want output to
                go to this directory by default?  It should go to cwd
      
     static  -- important files in here, however, wrfsi.nl contains
                &filetimespec information which is definitely not
      "static" -- it changes with each new forecast.
     wrfprd  -- get rid of this entirely.

 3) prior to running wrfsi.pl, it is very difficult to determine how
  the code is going to try to find model GRIB files.  Where is this
  specified?  We store GRIB files in one central disk which is
  readable but not writable by me.  Do I need to create links
  somewhere?  -- this turns out to be specified in
  /home/disk/wrf1/mm5rt/wrf_tests/extdataroot/static/grib_prep.nl is
  that written by one of these programs or am I just supposed to know
  that I should edit it?

 x) A problem with all of WRF SI, input and output should by default
  be into the current working directory unless otherwise specified by
  options on the command-line.  There are way too many directories and
  environment variables floating around to keep track of.
  Also, namelists that are going to be used by a program should be in
  the current directory or specified on the command line.

  x) Purging should be done by separate routines.  Having wrfsi.pl
  purge a directory such as this
Purging /home/disk/wrf1/mm5rt/wrf_tests/extdataroot/extprd/GFS:2003-07-06_12
  is not helpful when you are simple trying to set up a run.

  x) When some routine dies such as in wrfprep.pl line 474
...
  }else{
    print WPLOG "No matching files found for initial conditions.\n";
    $timenow = `date -u`; chomp $timenow;
    print "Died at $timenow\n";
    close (WPLOG);
    die;
  }
...
  Not only the log, but stdout should indicate what files could not be
  found. 

  x) work directories, such as $EXT_DATAROOT/work/xx should be local
  to the machine on which the code is running.  /tmp is ideal for this
  on most platforms.  At least allow this as an option to wrfsi.pl,
  grid_prep.pl, or wrfprep.pl.


see HOW_TO_RUN.txt, README, INSTALL, 

in /home/disk/wrf1/mm5rt/wrf_tests/WRFV1/setenv.csh
  setenv NETCDF /usr/src/netcdf-3.5.0
  setenv DATAROOT /home/disk/wrf1/mm5rt/wrf_tests/dataroot
  setenv GEOG_DATAROOT /home/disk/wrf1/mm5rt/wrf_tests/terrain
  setenv EXT_DATAROOT /home/disk/wrf1/mm5rt/wrf_tests/extdataroot
  setenv INSTALLROOT /home/disk/wrf1/mm5rt/wrf_tests/WRFV1/wrfsi
  setenv PERL5OPT -Mblib=/home/disk/wrf1/mm5rt/wrf_tests/WRFV1/wrfsi/gui/perlTk/blib
  setenv MOAD_DATAROOT /home/disk/wrf1/mm5rt/wrf_tests/dataroot/mm5d1aug02


7/3/2003: working on wrf:
 . got wrfsi and wrf.exe and real.exe compiled
 . on my domain, grep floating exception around 19th timestep on wrf18
   in /tmp/
 . my executable runs fine after running real.exe on Ryan's
   real_input* files ---> implication, my wrfsi is fouled up
 . next step is to try to run wrfsi to create his domain -- not sure
   how to set environment variables to created this new domain
 . I am not sure that their OpenMP compiler flags work, you see some
   pgf90 lines that definitely do not have "-mp" in them, not sure if
   they should be there -- tried one pretty standard version of
   their's and got the following for Ryan's domain in /tmp/ryans on wrf18:
       INPUT LANDUSE = USGS
       LANDUSE TYPE = USGS FOUND           24  CATEGORIES            2  SEASONS
       WATER CATEGORY =            16  SNOW CATEGORY =            24
       Segmentation fault
   Then, I made sure configure.wrf had
       FCBASEOPTS      =       -fastsse -byteswapio -Ktrap=fp -Mfree \
        -tp p7 -pc 32 -Mvect=prefetch,cachesize:524288 -mp $(FCDEBUG) 
   and this resulted in a successful running of the thing but with
   Timings showing "0 f0.00000 elapsed seconds" for everthing.
   Top showed one wrf.exe but timing was 1:48 vs. 4:12 (no -fastsse)
   and  3:11 (with -fastsse and -mp code, but with OMP_NUM_THREADS=1)
   .. timing indicates this is working
   .. wrfout files are in exact agreement

 . Ryan's stuff can be found in:
    /home/disk/ridge/torn/wrfsi/data/static/wrfsi.nl
    /home/disk/ridge/torn/wrfsi/data/siprd/  -- daily real_input* files
    /home/disk/ridge/torn/WRFV1/test/em_real/ -- his run directory, namelist.input
    /home/disk/ridge/torn/wrf_scripts/WRF_RT.OUT -- stdout from his run
 . Ran the 2-d squall line case seemingly fine on wrf16 even though
   the output showed:
     Timing for main: time step #    1199 on domain   1:    0.00000 elapsed seconds.
     Timing for main: time step #    1200 on domain   1:    0.00000 elapsed seconds.
     Timing for main: time step #    1200 on domain   1:    0.00000 elapsed seconds.
     Timing for Writing output for domain        1:    0.00000 elapsed seconds.


setting up a non-centered domain:
> I have a couple of questions about using the wrfsi GUI.  
> 
> #1 Since there is no nesting capability, it looks like I cannot set up
>    a 36-km domain to match our current MM5 real-time 36-km domain
>    (http://www.atmos.washington.edu/mm5rt/36kmterrain.gif) which was
>    created by running MM5 TERRAIN and using a bigger outer 36-km
>    domain that is never used.  We chose to do this so that the
>    longitude line over Washington would be vertical and not skewed.
>    Since WRF seems to keep that requirement of center lat/lon being in
>    the center of the mother domain and there are no nests, I am stuck
>    creating a single bigger domain or a skewed version.  Is that
>    correct?

Actually SI allows you to define a off-centered grid. The variables you 
need to define are 

MOAD_KNOWN_LAT and MOAD_KNOWN_LON: these are the 
lat and long of your center grid point (in your case, the 36 km domain's
center value); 

MOAD_STAND_LATS: these are the true lats as you would
use it in MM5; 

MOAD_STAND_LONS: this is the center longitude
from the MM5 definition (and this doesn't have to be the same as the
longitude of your center grid point). 

So what you need to find are the values of lat and long in your center 36 km domain.
> 
> #2 I am on the "Initialization Controls" tab of the wrfsi GUI and I
>    really want to know what all of the NUM_DOMAINS, DOMAIN_ID_LIST,
>    HINTERP_METHOD, etc. mean and what my choices are.  Where can I
>    find out about these?

There is a README.wrfsi_nl in the top wrfsi directory, and that has explanation
of the namelist variables used for SI.


> 
here's how some people have compiled WRF with ifort Intel fortran
compiler:
From: "Hara, Masayuki" <hara.masayuki@jamstec.go.jp>
I compiled WRFSI with ifort v8.1 on i686/Redhat 7.3.
When you compile WRFSI using ifort, NetCDF libraries should be compiled
with gcc/ifort, too.
  
#--------------------------------------------------
CC=gcc
DEFS=-Di686 -DDEC -DBIT32 -DF90 -DFORTRANUNDERSCORE -DDYNAMIC
CFLAGS= $(DEFS)
DBCFLAGS= -g -DDEBUG=1 $(DEFS)
FC       = ifort
NCARGFC  = /usr/local/ncarg/bin/ncargf77
FFLAGS =  $(INC) -O2 -convert big_endian
DBFLAGS = $(INC) -g -C -convert big_endian
FIXED=-fixed
FREE=-free
LDFLAGS=
CPP =  ifort -E -free
CPPFLAGS = $(INC) $(DEFS)
USECPP=1
MACHTYPE = pc
#--------------------------------------------------