<core_client_version>7.16.6</core_client_version>
<![CDATA[
<message>
process exited with code 1 (0x1, -255)</message>
<stderr_txt>
k.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [restore_default_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [trapfpe_treatment@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:364212:364212] [20221217:233931:1671316771:0.001] [catch_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
23:40:51 STEP 48 H= 12:00 +CPU= 69.467
23:42:00 STEP 49 H= 12:15 +CPU= 68.033
23:43:09 STEP 50 H= 12:30 +CPU= 69.219
23:44:20 STEP 51 H= 12:45 +CPU= 70.302
23:45:35 STEP 52 H= 13:00 +CPU= 72.622
23:46:44 STEP 53 H= 13:15 +CPU= 69.437
23:47:54 STEP 54 H= 13:30 +CPU= 69.373
23:49:03 STEP 55 H= 13:45 +CPU= 68.150
23:54:35 STEP 56 H= 14:00 +CPU= 69.611
23:55:43 STEP 57 H= 14:15 +CPU= 68.264
23:56:52 STEP 58 H= 14:30 +CPU= 68.827
23:58:00 STEP 59 H= 14:45 +CPU= 67.091
23:59:27 STEP 60 H= 15:00 +CPU= 87.042
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000060
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000060
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000060
00:00:38 STEP 61 H= 15:15 +CPU= 70.694
00:01:47 STEP 62 H= 15:30 +CPU= 68.469
00:02:56 STEP 63 H= 15:45 +CPU= 68.031
00:04:07 STEP 64 H= 16:00 +CPU= 70.440
00:05:16 STEP 65 H= 16:15 +CPU= 69.054
01:31:27 STEP 66 H= 16:30 +CPU= 94.503
01:32:35 STEP 67 H= 16:45 +CPU= 67.247
01:33:45 STEP 68 H= 17:00 +CPU= 70.047
01:34:53 STEP 69 H= 17:15 +CPU= 67.969
01:36:01 STEP 70 H= 17:30 +CPU= 67.812
01:37:07 STEP 71 H= 17:45 +CPU= 66.703
01:38:32 STEP 72 H= 18:00 +CPU= 84.602
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000072
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000072
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000072
01:39:42 STEP 73 H= 18:15 +CPU= 68.992
01:40:51 STEP 74 H= 18:30 +CPU= 69.240
01:42:01 STEP 75 H= 18:45 +CPU= 69.445
01:43:09 STEP 76 H= 19:00 +CPU= 68.763
01:44:17 STEP 77 H= 19:15 +CPU= 67.104
01:45:26 STEP 78 H= 19:30 +CPU= 69.141
01:46:33 STEP 79 H= 19:45 +CPU= 67.400
01:47:44 STEP 80 H= 20:00 +CPU= 70.868
01:48:53 STEP 81 H= 20:15 +CPU= 65.921
01:50:00 STEP 82 H= 20:30 +CPU= 66.735
01:51:09 STEP 83 H= 20:45 +CPU= 68.524
01:52:34 STEP 84 H= 21:00 +CPU= 85.045
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000084
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000084
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000084
01:53:41 STEP 85 H= 21:15 +CPU= 67.002
01:54:50 STEP 86 H= 21:30 +CPU= 68.146
01:55:56 STEP 87 H= 21:45 +CPU= 66.933
01:57:05 STEP 88 H= 22:00 +CPU= 68.549
01:58:13 STEP 89 H= 22:15 +CPU= 67.652
01:59:20 STEP 90 H= 22:30 +CPU= 66.968
02:00:27 STEP 91 H= 22:45 +CPU= 67.408
02:01:37 STEP 92 H= 23:00 +CPU= 68.896
02:02:46 STEP 93 H= 23:15 +CPU= 68.960
02:03:54 STEP 94 H= 23:30 +CPU= 68.145
02:05:02 STEP 95 H= 23:45 +CPU= 67.691
02:06:36 STEP 96 H= 24:00 +CPU= 89.972
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000096
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000096
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000096
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000000
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000000
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000000
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000012
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000012
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000012
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000024
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000024
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000024
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000036
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000036
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000036
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000048
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000048
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000048
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000060
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000060
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000060
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000072
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000072
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000072
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000084
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000084
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000084
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_a037_2016092300_15_949_12166509_0_r1617716062_0.zip
Uploading the intermediate file: upload_file_0.zip
02:07:43 STEP 97 H= 24:15 +CPU= 66.672
02:08:51 STEP 98 H= 24:30 +CPU= 67.243
02:09:59 STEP 99 H= 24:45 +CPU= 68.161
02:11:08 STEP 100 H= 25:00 +CPU= 69.007
02:12:17 STEP 101 H= 25:15 +CPU= 69.578
02:13:25 STEP 102 H= 25:30 +CPU= 67.326
02:14:32 STEP 103 H= 25:45 +CPU= 67.182
02:15:40 STEP 104 H= 26:00 +CPU= 68.195
02:16:49 STEP 105 H= 26:15 +CPU= 68.355
02:17:58 STEP 106 H= 26:30 +CPU= 69.182
02:19:07 STEP 107 H= 26:45 +CPU= 67.351
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_bl_1.07_x86_64-pc-linux-gnu
(argv1) start_date: 2016092300
(argv2) exptid: h7zg
(argv3) unique_member_id: a037
(argv4) batchid: 949
(argv5) wuid: 12166509
(argv6) fclen: 15
(argv7) app_name: oifs_43r3_bl
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/0
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_bl
version: 1.07
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/0/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/0/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_04cf6b7969dbfe9db4de14a6c6961598 to: /var/lib/boinc-client/slots/0/oifs_43r3_bl_a037_2016092300_15_949_12166509.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/0/oifs_43r3_bl_a037_2016092300_15_949_12166509.zip
ic_ancil_file: ic_ancil_12166509
ifsdata_file: ifsdata_12166509
climate_data_file: clim_data_12166509
horiz_resolution: 159
vert_resolution: 91
grid_type: l_2
upload_interval: 96
utstep: 900
nfrres: restart dump frequency (steps) 48
Copying IC ancils from: ../../projects/climateprediction.net/jf_f7ecfd27aa1e68bbe7e8d19800e25c73 to: /var/lib/boinc-client/slots/0/ic_ancil_12166509.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/0/ic_ancil_12166509.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_3e4769df112a7ee66b1eaa0c599c34ab to: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12166509.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12166509.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/0/159l_2/clim_data_12166509.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/0/159l_2/clim_data_12166509.zip
Checking for progress XML file: /var/lib/boinc-client/slots/0/progress_file_12166509.xml
Opened progress file ok : /var/lib/boinc-client/slots/0/progress_file_12166509.xml
-- Model is restarting --
Adjusting last_iter, 108, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/0/progress_file_12166509.xml
last_cpu_time: 9706
upload_file_number: 1
last_iter: 95
last_upload: 86400
model_completed: 0
total_length_of_simulation: 1296000
result_base_name: oifs_43r3_bl_a037_2016092300_15_949_12166509_0_r1617716062
The child process has been launched with process id: 365073
Executing the command: /var/lib/boinc-client/slots/0/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2091] fp = 0x30eaa80
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [ignore_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [restore_default_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [trapfpe_treatment@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365073:365073] [20221218:022652:1671326812:0.001] [catch_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
02:28:16 STEP 96 H= 24:00 +CPU= 72.573
02:29:24 STEP 97 H= 24:15 +CPU= 67.158
02:30:30 STEP 98 H= 24:30 +CPU= 66.235
02:31:38 STEP 99 H= 24:45 +CPU= 67.034
02:32:49 STEP 100 H= 25:00 +CPU= 69.935
02:33:58 STEP 101 H= 25:15 +CPU= 68.482
02:35:06 STEP 102 H= 25:30 +CPU= 67.928
02:36:16 STEP 103 H= 25:45 +CPU= 67.374
02:37:29 STEP 104 H= 26:00 +CPU= 72.959
02:38:30 STEP 105 H= 26:15 +CPU= 60.439
02:39:36 STEP 106 H= 26:30 +CPU= 66.284
02:40:47 STEP 107 H= 26:45 +CPU= 70.071
02:42:13 STEP 108 H= 27:00 +CPU= 84.299
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000108
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000108
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000108
02:43:19 STEP 109 H= 27:15 +CPU= 65.770
02:44:26 STEP 110 H= 27:30 +CPU= 66.367
02:45:33 STEP 111 H= 27:45 +CPU= 67.078
02:46:46 STEP 112 H= 28:00 +CPU= 71.672
02:47:48 STEP 113 H= 28:15 +CPU= 61.781
02:49:00 STEP 114 H= 28:30 +CPU= 71.167
02:50:10 STEP 115 H= 28:45 +CPU= 68.833
02:51:20 STEP 116 H= 29:00 +CPU= 66.900
02:52:25 STEP 117 H= 29:15 +CPU= 65.342
02:53:33 STEP 118 H= 29:30 +CPU= 66.994
02:54:42 STEP 119 H= 29:45 +CPU= 68.413
02:56:08 STEP 120 H= 30:00 +CPU= 84.653
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000120
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000120
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000120
02:57:10 STEP 121 H= 30:15 +CPU= 61.433
02:58:16 STEP 122 H= 30:30 +CPU= 65.634
02:59:23 STEP 123 H= 30:45 +CPU= 66.551
03:00:39 STEP 124 H= 31:00 +CPU= 74.818
03:01:48 STEP 125 H= 31:15 +CPU= 68.625
03:03:01 STEP 126 H= 31:30 +CPU= 72.478
03:04:14 STEP 127 H= 31:45 +CPU= 67.269
03:05:23 STEP 128 H= 32:00 +CPU= 66.732
03:06:28 STEP 129 H= 32:15 +CPU= 64.790
03:07:35 STEP 130 H= 32:30 +CPU= 66.602
03:08:50 STEP 131 H= 32:45 +CPU= 74.140
03:10:17 STEP 132 H= 33:00 +CPU= 86.134
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000132
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000132
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000132
03:11:24 STEP 133 H= 33:15 +CPU= 65.163
03:12:33 STEP 134 H= 33:30 +CPU= 68.123
03:13:42 STEP 135 H= 33:45 +CPU= 68.466
03:14:52 STEP 136 H= 34:00 +CPU= 69.147
03:15:59 STEP 137 H= 34:15 +CPU= 66.748
03:17:08 STEP 138 H= 34:30 +CPU= 68.552
03:18:17 STEP 139 H= 34:45 +CPU= 68.318
03:19:29 STEP 140 H= 35:00 +CPU= 71.043
03:20:37 STEP 141 H= 35:15 +CPU= 67.325
03:21:46 STEP 142 H= 35:30 +CPU= 68.170
03:22:54 STEP 143 H= 35:45 +CPU= 67.591
03:24:27 STEP 144 H= 36:00 +CPU= 90.098
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000144
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000144
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000144
03:25:33 STEP 145 H= 36:15 +CPU= 66.190
03:26:41 STEP 146 H= 36:30 +CPU= 67.284
03:27:48 STEP 147 H= 36:45 +CPU= 66.174
03:28:57 STEP 148 H= 37:00 +CPU= 68.778
03:30:04 STEP 149 H= 37:15 +CPU= 66.314
03:31:14 STEP 150 H= 37:30 +CPU= 66.795
03:32:30 STEP 151 H= 37:45 +CPU= 73.544
03:33:38 STEP 152 H= 38:00 +CPU= 67.890
03:34:42 STEP 153 H= 38:15 +CPU= 63.864
03:35:50 STEP 154 H= 38:30 +CPU= 67.536
03:36:58 STEP 155 H= 38:45 +CPU= 67.313
03:38:24 STEP 156 H= 39:00 +CPU= 84.487
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000156
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000156
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000156
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_bl_1.07_x86_64-pc-linux-gnu
(argv1) start_date: 2016092300
(argv2) exptid: h7zg
(argv3) unique_member_id: a037
(argv4) batchid: 949
(argv5) wuid: 12166509
(argv6) fclen: 15
(argv7) app_name: oifs_43r3_bl
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/0
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_bl
version: 1.07
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/0/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/0/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_04cf6b7969dbfe9db4de14a6c6961598 to: /var/lib/boinc-client/slots/0/oifs_43r3_bl_a037_2016092300_15_949_12166509.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/0/oifs_43r3_bl_a037_2016092300_15_949_12166509.zip
ic_ancil_file: ic_ancil_12166509
ifsdata_file: ifsdata_12166509
climate_data_file: clim_data_12166509
horiz_resolution: 159
vert_resolution: 91
grid_type: l_2
upload_interval: 96
utstep: 900
nfrres: restart dump frequency (steps) 48
Copying IC ancils from: ../../projects/climateprediction.net/jf_f7ecfd27aa1e68bbe7e8d19800e25c73 to: /var/lib/boinc-client/slots/0/ic_ancil_12166509.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/0/ic_ancil_12166509.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_3e4769df112a7ee66b1eaa0c599c34ab to: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12166509.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12166509.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/0/159l_2/clim_data_12166509.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/0/159l_2/clim_data_12166509.zip
Checking for progress XML file: /var/lib/boinc-client/slots/0/progress_file_12166509.xml
Opened progress file ok : /var/lib/boinc-client/slots/0/progress_file_12166509.xml
-- Model is restarting --
Adjusting last_iter, 157, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/0/progress_file_12166509.xml
last_cpu_time: 13964
upload_file_number: 1
last_iter: 143
last_upload: 86400
model_completed: 0
total_length_of_simulation: 1296000
result_base_name: oifs_43r3_bl_a037_2016092300_15_949_12166509_0_r1617716062
The child process has been launched with process id: 365813
Executing the command: /var/lib/boinc-client/slots/0/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2091] fp = 0x30eaa80
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [ignore_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [restore_default_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [trapfpe_treatment@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:365813:365813] [20221218:035059:1671331859:0.001] [catch_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
03:52:27 STEP 144 H= 36:00 +CPU= 73.591
03:53:34 STEP 145 H= 36:15 +CPU= 66.891
03:54:41 STEP 146 H= 36:30 +CPU= 66.609
03:55:49 STEP 147 H= 36:45 +CPU= 67.540
03:56:58 STEP 148 H= 37:00 +CPU= 67.813
03:58:02 STEP 149 H= 37:15 +CPU= 63.368
03:59:13 STEP 150 H= 37:30 +CPU= 69.477
04:00:23 STEP 151 H= 37:45 +CPU= 69.654
04:01:31 STEP 152 H= 38:00 +CPU= 66.614
04:02:38 STEP 153 H= 38:15 +CPU= 66.657
04:03:46 STEP 154 H= 38:30 +CPU= 67.829
04:04:53 STEP 155 H= 38:45 +CPU= 66.160
04:06:15 STEP 156 H= 39:00 +CPU= 81.795
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000156
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000156
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000156
04:07:24 STEP 157 H= 39:15 +CPU= 68.432
04:08:32 STEP 158 H= 39:30 +CPU= 67.749
04:09:36 STEP 159 H= 39:45 +CPU= 63.117
04:10:47 STEP 160 H= 40:00 +CPU= 70.258
04:11:56 STEP 161 H= 40:15 +CPU= 68.725
04:13:04 STEP 162 H= 40:30 +CPU= 68.038
04:14:11 STEP 163 H= 40:45 +CPU= 66.970
04:15:19 STEP 164 H= 41:00 +CPU= 67.559
04:16:27 STEP 165 H= 41:15 +CPU= 68.411
04:17:34 STEP 166 H= 41:30 +CPU= 66.435
04:18:40 STEP 167 H= 41:45 +CPU= 65.996
04:20:07 STEP 168 H= 42:00 +CPU= 86.296
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000168
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000168
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000168
04:21:14 STEP 169 H= 42:15 +CPU= 64.463
04:22:23 STEP 170 H= 42:30 +CPU= 68.487
04:23:30 STEP 171 H= 42:45 +CPU= 66.493
04:24:39 STEP 172 H= 43:00 +CPU= 69.326
04:25:47 STEP 173 H= 43:15 +CPU= 67.323
04:26:48 STEP 174 H= 43:30 +CPU= 60.581
04:27:53 STEP 175 H= 43:45 +CPU= 64.781
04:29:02 STEP 176 H= 44:00 +CPU= 69.016
04:30:10 STEP 177 H= 44:15 +CPU= 67.532
04:31:17 STEP 178 H= 44:30 +CPU= 65.934
04:32:22 STEP 179 H= 44:45 +CPU= 63.974
04:33:49 STEP 180 H= 45:00 +CPU= 85.906
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000180
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000180
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000180
04:34:59 STEP 181 H= 45:15 +CPU= 68.026
04:36:15 STEP 182 H= 45:30 +CPU= 67.203
04:37:24 STEP 183 H= 45:45 +CPU= 68.292
04:38:39 STEP 184 H= 46:00 +CPU= 72.990
04:39:48 STEP 185 H= 46:15 +CPU= 68.447
04:40:57 STEP 186 H= 46:30 +CPU= 67.873
04:42:03 STEP 187 H= 46:45 +CPU= 65.716
04:43:07 STEP 188 H= 47:00 +CPU= 63.822
04:44:16 STEP 189 H= 47:15 +CPU= 68.872
04:45:23 STEP 190 H= 47:30 +CPU= 66.531
04:46:27 STEP 191 H= 47:45 +CPU= 64.486
04:47:57 STEP 192 H= 48:00 +CPU= 87.970
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000192
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000192
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000192
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000096
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000096
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000096
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000108
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000108
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000108
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000120
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000120
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000120
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000132
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000132
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000132
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000144
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000144
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000144
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000156
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000156
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000156
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000168
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000168
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000168
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMGGh7zg+000180
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMSHh7zg+000180
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509/ICMUAh7zg+000180
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_a037_2016092300_15_949_12166509_0_r1617716062_1.zip
Uploading the intermediate file: upload_file_1.zip
04:49:03 STEP 193 H= 48:15 +CPU= 64.141
04:50:07 STEP 194 H= 48:30 +CPU= 64.429
04:51:15 STEP 195 H= 48:45 +CPU= 67.345
04:52:23 STEP 196 H= 49:00 +CPU= 67.321
04:53:30 STEP 197 H= 49:15 +CPU= 66.465
04:54:38 STEP 198 H= 49:30 +CPU= 67.792
04:55:45 STEP 199 H= 49:45 +CPU= 65.977
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_bl_1.07_x86_64-pc-linux-gnu
(argv1) start_date: 2016092300
(argv2) exptid: h7zg
(argv3) unique_member_id: a037
(argv4) batchid: 949
(argv5) wuid: 12166509
(argv6) fclen: 15
(argv7) app_name: oifs_43r3_bl
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/0
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_bl
version: 1.07
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/0/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/0/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_04cf6b7969dbfe9db4de14a6c6961598 to: /var/lib/boinc-client/slots/0/oifs_43r3_bl_a037_2016092300_15_949_12166509.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/0/oifs_43r3_bl_a037_2016092300_15_949_12166509.zip
ic_ancil_file: ic_ancil_12166509
ifsdata_file: ifsdata_12166509
climate_data_file: clim_data_12166509
horiz_resolution: 159
vert_resolution: 91
grid_type: l_2
upload_interval: 96
utstep: 900
nfrres: restart dump frequency (steps) 48
Copying IC ancils from: ../../projects/climateprediction.net/jf_f7ecfd27aa1e68bbe7e8d19800e25c73 to: /var/lib/boinc-client/slots/0/ic_ancil_12166509.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/0/ic_ancil_12166509.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_3e4769df112a7ee66b1eaa0c599c34ab to: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12166509.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12166509.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/0/159l_2/clim_data_12166509.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/0/159l_2/clim_data_12166509.zip
Checking for progress XML file: /var/lib/boinc-client/slots/0/progress_file_12166509.xml
Opened progress file ok : /var/lib/boinc-client/slots/0/progress_file_12166509.xml
-- Model is restarting --
Adjusting last_iter, 200, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/0/progress_file_12166509.xml
last_cpu_time: 17813
upload_file_number: 2
last_iter: 191
last_upload: 172800
model_completed: 0
total_length_of_simulation: 1296000
result_base_name: oifs_43r3_bl_a037_2016092300_15_949_12166509_0_r1617716062
The child process has been launched with process id: 366044
Executing the command: /var/lib/boinc-client/slots/0/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2091] fp = 0x30eaa80
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [ignore_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [restore_default_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [trapfpe_treatment@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366044:366044] [20221218:050035:1671336035:0.001] [catch_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
05:01:58 STEP 192 H= 48:00 +CPU= 72.059
05:03:08 STEP 193 H= 48:15 +CPU= 69.938
05:04:16 STEP 194 H= 48:30 +CPU= 68.226
05:05:25 STEP 195 H= 48:45 +CPU= 68.424
05:06:35 STEP 196 H= 49:00 +CPU= 69.306
05:07:44 STEP 197 H= 49:15 +CPU= 69.047
05:08:53 STEP 198 H= 49:30 +CPU= 68.360
05:10:03 STEP 199 H= 49:45 +CPU= 70.351
05:11:14 STEP 200 H= 50:00 +CPU= 70.493
05:12:22 STEP 201 H= 50:15 +CPU= 68.501
05:13:31 STEP 202 H= 50:30 +CPU= 69.047
05:14:40 STEP 203 H= 50:45 +CPU= 68.919
05:16:05 STEP 204 H= 51:00 +CPU= 84.989
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000204
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000204
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000204
05:17:12 STEP 205 H= 51:15 +CPU= 66.976
05:18:22 STEP 206 H= 51:30 +CPU= 69.079
05:19:31 STEP 207 H= 51:45 +CPU= 69.229
05:20:43 STEP 208 H= 52:00 +CPU= 71.723
05:21:51 STEP 209 H= 52:15 +CPU= 68.315
05:23:00 STEP 210 H= 52:30 +CPU= 69.155
05:24:09 STEP 211 H= 52:45 +CPU= 68.403
05:25:19 STEP 212 H= 53:00 +CPU= 68.457
05:26:28 STEP 213 H= 53:15 +CPU= 68.961
05:27:36 STEP 214 H= 53:30 +CPU= 68.144
05:28:44 STEP 215 H= 53:45 +CPU= 67.634
05:30:11 STEP 216 H= 54:00 +CPU= 84.472
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000216
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000216
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000216
05:31:17 STEP 217 H= 54:15 +CPU= 66.543
05:32:29 STEP 218 H= 54:30 +CPU= 69.718
05:33:39 STEP 219 H= 54:45 +CPU= 69.175
05:34:49 STEP 220 H= 55:00 +CPU= 70.662
05:35:59 STEP 221 H= 55:15 +CPU= 69.932
05:37:10 STEP 222 H= 55:30 +CPU= 68.883
05:38:19 STEP 223 H= 55:45 +CPU= 68.717
05:39:30 STEP 224 H= 56:00 +CPU= 71.599
05:40:40 STEP 225 H= 56:15 +CPU= 69.474
05:41:50 STEP 226 H= 56:30 +CPU= 70.084
05:43:00 STEP 227 H= 56:45 +CPU= 69.642
05:44:28 STEP 228 H= 57:00 +CPU= 88.624
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000228
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000228
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000228
05:45:36 STEP 229 H= 57:15 +CPU= 67.495
05:46:45 STEP 230 H= 57:30 +CPU= 68.598
05:47:54 STEP 231 H= 57:45 +CPU= 68.779
05:49:04 STEP 232 H= 58:00 +CPU= 70.487
05:50:13 STEP 233 H= 58:15 +CPU= 68.262
05:51:23 STEP 234 H= 58:30 +CPU= 69.553
05:52:33 STEP 235 H= 58:45 +CPU= 69.533
05:53:45 STEP 236 H= 59:00 +CPU= 71.993
05:54:53 STEP 237 H= 59:15 +CPU= 68.282
05:56:01 STEP 238 H= 59:30 +CPU= 67.392
05:57:09 STEP 239 H= 59:45 +CPU= 68.236
05:58:45 STEP 240 H= 60:00 +CPU= 94.560
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGh7zg+000240
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHh7zg+000240
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAh7zg+000240
05:59:52 STEP 241 H= 60:15 +CPU= 67.299
06:00:55 STEP 242 H= 60:30 +CPU= 62.533
06:02:03 STEP 243 H= 60:45 +CPU= 67.924
06:03:13 STEP 244 H= 61:00 +CPU= 70.080
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_bl_1.07_x86_64-pc-linux-gnu
(argv1) start_date: 2016092300
(argv2) exptid: h7zg
(argv3) unique_member_id: a037
(argv4) batchid: 949
(argv5) wuid: 12166509
(argv6) fclen: 15
(argv7) app_name: oifs_43r3_bl
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/0
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_bl
version: 1.07
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_12166509
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/0/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/0/oifs_43r3_bl_app_1.07_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_04cf6b7969dbfe9db4de14a6c6961598 to: /var/lib/boinc-client/slots/0/oifs_43r3_bl_a037_2016092300_15_949_12166509.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/0/oifs_43r3_bl_a037_2016092300_15_949_12166509.zip
ic_ancil_file: ic_ancil_12166509
ifsdata_file: ifsdata_12166509
climate_data_file: clim_data_12166509
horiz_resolution: 159
vert_resolution: 91
grid_type: l_2
upload_interval: 96
utstep: 900
nfrres: restart dump frequency (steps) 48
Copying IC ancils from: ../../projects/climateprediction.net/jf_f7ecfd27aa1e68bbe7e8d19800e25c73 to: /var/lib/boinc-client/slots/0/ic_ancil_12166509.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/0/ic_ancil_12166509.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_3e4769df112a7ee66b1eaa0c599c34ab to: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12166509.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12166509.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/0/159l_2/clim_data_12166509.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/0/159l_2/clim_data_12166509.zip
Checking for progress XML file: /var/lib/boinc-client/slots/0/progress_file_12166509.xml
Opened progress file ok : /var/lib/boinc-client/slots/0/progress_file_12166509.xml
-- Model is restarting --
Adjusting last_iter, 245, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/0/progress_file_12166509.xml
last_cpu_time: 21577
upload_file_number: 2
last_iter: 239
last_upload: 172800
model_completed: 0
total_length_of_simulation: 1296000
result_base_name: oifs_43r3_bl_a037_2016092300_15_949_12166509_0_r1617716062
The child process has been launched with process id: 366743
Executing the command: /var/lib/boinc-client/slots/0/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2091] fp = 0x30eaa80
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.000] [process_options@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [ignore_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [restore_default_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [trapfpe_treatment@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x80f7e0 (old at 0x401140)
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [signal_drhook_init@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x80f7e0 (old at (nil))
[EC_DRHOOK:B20:1:1:366743:366743] [20221218:060707:1671340027:0.001] [catch_signals@/home/abowery/Desktop/OpenIFS/oifs_43r3_bl/gc_oifs43r3/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
06:08:30 STEP 240 H= 60:00 +CPU= 71.295
06:09:41 STEP 241 H= 60:15 +CPU= 70.056
06:10:50 STEP 242 H= 60:30 +CPU= 68.219
The child process terminated with status: 0
..Failed, model did not complete successfully
</stderr_txt>
]]>
