{
    "titan_aprun": {
        "description"                 : "The Cray XK7 supercomputer located at the Oak Ridge Leadership Computing Facility (OLCF), (https://www.olcf.ornl.gov/titan/)",
        "notes"                       : "Requires the use of an RSA SecurID on every connection.",
        "schemas"                     : ["local"],
        "local"                       : {
            "job_manager_hop"         : "fork://localhost/",
            "job_manager_endpoint"    : "torque://localhost/",
            "filesystem_endpoint"     : "file://localhost/"
        },
        "default_queue"               : "batch",
        "lrms"                        : "TORQUE",
        "lfs_per_node"                : "/tmp", 
        "agent_type"                  : "multicore",
        "agent_config"                : "cray_aprun",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "agent_launch_method"         : "APRUN",
        "task_launch_method"          : "APRUN",
        "mpi_launch_method"           : "APRUN",
        "pre_bootstrap_0"             : [
            "module unload PrgEnv-pgi",
            "module load PrgEnv-gnu",
            "module load xalt",
            "module load python",
            "module load python_pip",
            "module load python_virtualenv",
            # Workaround for ZMQ runtime failure
            "export LD_PRELOAD=/lib64/librt.so.1"
        ],
        "pre_bootstrap_1"             : [
          # "export PATH=$PATH",
          # "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH",
            "module load xalt"
        ],
        "default_remote_workdir"      : "$MEMBERWORK/`groups | cut -d' ' -f2`",
        "valid_roots"                 : ["/lustre/atlas/scratch"],
        "rp_version"                  : "local",
        "virtenv_mode"                : "use",
        "virtenv"                     : "/lustre/atlas2/csc230/world-shared/ve.rp.titan.20017.10.06",
        "stage_cacerts"               : true,
        "python_dist"                 : "default",
        "virtenv_dist"                : "system",
        "gpus_per_node"               : 1
     #  "saga_jd_supplement"          : {"candidate_hosts" : ["titan"]}
    },

    "rhea_aprun": {
        "description"                 : "The Cray XK7 supercomputer located at the Oak Ridge Leadership Computing Facility (OLCF), (https://www.olcf.ornl.gov/titan/)",
        "notes"                       : "Requires the use of an RSA SecurID on every connection.",
        "schemas"                     : ["local"],
        "local"                       : {
            "job_manager_endpoint"    : "torque://localhost",
            "filesystem_endpoint"     : "file://localhost/"
        },
        "default_queue"               : "batch",
        "lrms"                        : "TORQUE",
        "agent_type"                  : "multicore",
        "agent_config"                : "rhea",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "agent_launch_method"         : "APRUN",
        "task_launch_method"          : "APRUN",
        "mpi_launch_method"           : "APRUN",
        "pre_bootstrap_0"             : [
            "module purge",
            "module load PE-gnu",
            "module load python",
            "module load python_pip",
            "module load python_virtualenv",
            # Workaround for ZMQ runtime failure
            "export LD_PRELOAD=/lib64/librt.so.1"
        ],
        "default_remote_workdir"      : "$MEMBERWORK/`groups | cut -d' ' -f2`",
        "valid_roots"                 : ["/lustre/atlas/scratch"],
        "rp_version"                  : "local",
        "virtenv_mode"                : "use",
        "virtenv"                     : "/lustre/atlas2/csc230/world-shared/ve.rp.rhea.20017.08.07",
        "stage_cacerts"               : true,
        "python_dist"                 : "default",
        "virtenv_dist"                : "system",
        "gpus_per_node"               : 1,
        "saga_jd_supplement"          : {"candidate_hosts" : ["rhea"]},
        "forward_tunnel_endpoint"     : "rhea-login4g.ccs.ornl.gov"
    },

    "titan_orte": {
        "description"                 : "The Cray XK7 supercomputer located at the Oak Ridge Leadership Computing Facility (OLCF), (https://www.olcf.ornl.gov/titan/)",
        "notes"                       : "Requires the use of an RSA SecurID on every connection.",
        "schemas"                     : ["local"],
        "local"                       : {
            "job_manager_hop"         : "fork://localhost/",
            "job_manager_endpoint"    : "torque://localhost",
            "filesystem_endpoint"     : "file://localhost/"
        },
        "default_queue"               : "batch",
        "lrms"                        : "TORQUE",
        "lfs_per_node"                : "/tmp", 
        "agent_type"                  : "multicore",
        "agent_config"                : "cray",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "SHELLFS",
        "agent_launch_method"         : "ORTE",
        "task_launch_method"          : "ORTE",
        "mpi_launch_method"           : "ORTE",
        "cores_per_node"              : "16",
        "gpus_per_node"               : 1,
        "pre_bootstrap_0"             : [
            "module swap PrgEnv-pgi PrgEnv-gnu",
            "module load python",
            "module use --append /lustre/atlas/world-shared/csc230/openmpi/modules/",
            "module load openmpi/2018_10_24_539f71d",
            # Workaround for ZMQ runtime failure
            "export LD_PRELOAD=/lib64/librt.so.1"
        ],
     #  "default_remote_workdir"      : "$MEMBERWORK/%(pd.project)s",
        "default_remote_workdir"      : "$MEMBERWORK/`groups | cut -d' ' -f2`",
        "valid_roots"                 : ["/lustre/atlas/scratch"],
        "rp_version"                  : "local",
        "virtenv_mode"                : "use",
        "virtenv"                     : "/lustre/atlas2/csc230/world-shared/ve.rp.titan.20017.10.06",
        "stage_cacerts"               : true,
        "python_dist"                 : "default",
        "virtenv_dist"                : "system"
    },

    "titan_ortelib": {
        "description"                 : "The Cray XK7 supercomputer located at the Oak Ridge Leadership Computing Facility (OLCF), (https://www.olcf.ornl.gov/titan/)",
        "notes"                       : "Requires the use of an RSA SecurID on every connection.",
        "schemas"                     : ["local"],
        "local"                       : {
            "job_manager_hop"         : "fork://localhost/",
            "job_manager_endpoint"    : "torque://localhost",
            "filesystem_endpoint"     : "file://localhost/"
        },
        "default_queue"               : "batch",
        "lrms"                        : "TORQUE",
        "lfs_per_node"                : "/tmp", 
        "agent_type"                  : "multicore",
        "agent_config"                : "cray",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "ORTE",
        "agent_launch_method"         : "ORTE",
        "task_launch_method"          : "ORTE_LIB",
        "mpi_launch_method"           : "ORTE_LIB",
        "gpus_per_node"               : 1,
        "pre_bootstrap_0"             : [
            "module swap PrgEnv-pgi PrgEnv-gnu",
            "module load python",
            "module use --append /lustre/atlas/world-shared/csc230/openmpi/modules/",
            "module load openmpi/2018_07_26_539f71d",
            # Workaround for ZMQ runtime failure
            "export LD_PRELOAD=/lib64/librt.so.1"
        ],
        "default_remote_workdir"      : "$MEMBERWORK/`groups | cut -d' ' -f2`",
        "valid_roots"                 : ["/lustre/atlas/scratch"],
        "rp_version"                  : "local",
        "virtenv_mode"                : "use",
        "virtenv"                     : "/lustre/atlas2/csc230/world-shared/ve.rp.titan.20017.10.06",
        "stage_cacerts"               : true,
        "python_dist"                 : "default",
        "virtenv_dist"                : "system"
    },


    "rhea": {
        "description"                 : "The Cray XK7 supercomputer located at the Oak Ridge Leadership Computing Facility (OLCF), (https://www.olcf.ornl.gov/rhea/)",
        "notes"                       : "Requires the use of an RSA SecurID on every connection.",
        "schemas"                     : ["local", "ssh", "go"],
        "ssh"                         : {
            "job_manager_endpoint"    : "torque+ssh://rhea.ccs.ornl.gov",
            "filesystem_endpoint"     : "sftp://rhea.ccs.ornl.gov/"
        },
        "local"                       : {
            "job_manager_endpoint"    : "torque://localhost",
            "filesystem_endpoint"     : "file://localhost/"
        },
        "go"                          : {
            "job_manager_endpoint"    : "torque+ssh://rhea.ccs.ornl.gov",
            "filesystem_endpoint"     : "go://olcf#dtn/"
        },
        "default_queue"               : "batch",
        "cores_per_node"              : "16",
        "gpus_per_node"               : 1,
        "lrms"                        : "TORQUE",
        #"agent_type"                  : "multicore",
        #"agent_config"                : "mpirun",
        "agent_scheduler"             : "CONTINUOUS",
        "agent_spawner"               : "POPEN",
        "agent_launch_method"         : "SSH",
        "task_launch_method"          : "SSH",
        "mpi_launch_method"           : "MPIRUN",
        "pre_bootstrap_0"             : [
            "module load python",
            # Workaround for ZMQ runtime failure
            "export LD_PRELOAD=/lib64/librt.so.1"
        ],
        "default_remote_workdir"      : "$MEMBERWORK/`groups | cut -d' ' -f2`",
        "valid_roots"                 : ["/lustre/atlas/scratch"],
        "rp_version"                  : "local",
        "virtenv_mode"                : "create",
        "stage_cacerts"               : true,
        "python_dist"                 : "default",
        "virtenv_dist"                : "system"
    }
}