<core_client_version>7.20.5</core_client_version>
<![CDATA[
<message>
Disk usage limit exceeded</message>
<stderr_txt>
98] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3124:3124] [20230126:014805:1674697685:0.002] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
01:56:25 STEP 2616 H=2616:00 +CPU= 82.709
02:00:49 STEP 2617 H=2617:00 +CPU= 45.581
02:05:32 STEP 2618 H=2618:00 +CPU= 47.061
02:12:37 STEP 2619 H=2619:00 +CPU= 71.715
02:17:24 STEP 2620 H=2620:00 +CPU= 47.985
02:22:19 STEP 2621 H=2621:00 +CPU= 48.223
02:29:35 STEP 2622 H=2622:00 +CPU= 73.377
02:32:55 STEP 2623 H=2623:00 +CPU= 36.603
02:37:13 STEP 2624 H=2624:00 +CPU= 42.714
02:44:33 STEP 2625 H=2625:00 +CPU= 73.184
02:49:10 STEP 2626 H=2626:00 +CPU= 46.067
02:54:00 STEP 2627 H=2627:00 +CPU= 47.973
03:02:10 STEP 2628 H=2628:00 +CPU= 82.193
Moving to projects directory: /var/lib/boinc/slots/50/ICMGGhq0f+002628
Moving to projects directory: /var/lib/boinc/slots/50/ICMSHhq0f+002628
Moving to projects directory: /var/lib/boinc/slots/50/ICMUAhq0f+002628
03:07:11 STEP 2629 H=2629:00 +CPU= 50.640
03:12:33 STEP 2630 H=2630:00 +CPU= 53.428
03:20:07 STEP 2631 H=2631:00 +CPU= 75.900
03:25:08 STEP 2632 H=2632:00 +CPU= 51.708
03:29:25 STEP 2633 H=2633:00 +CPU= 44.882
Quit request received from BOINC client, ending the child process
..Failed, model did not complete successfully
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2006050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0657
(argv4) batchid: 975
(argv5) wuid: 12192301
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc/slots/50
Project directory is: /var/lib/boinc/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301
..mkdir for temp folder for results failed
Copying: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc/slots/50/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc/slots/50/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_3f7806fb17fbdd54baa09aa36b4c4dc2 to: /var/lib/boinc/slots/50/oifs_43r3_ps_0657_2006050100_123_975_12192301.zip
Unzipping the namelist zip file: /var/lib/boinc/slots/50/oifs_43r3_ps_0657_2006050100_123_975_12192301.zip
ic_ancil_file: ic_ancil_12192301
ifsdata_file: ifsdata_12192301
climate_data_file: clim_data_12192301
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_271e89d9461af3fff859695ac7f7ded8 to: /var/lib/boinc/slots/50/ic_ancil_12192301.zip
Unzipping the IC ancils zip file: /var/lib/boinc/slots/50/ic_ancil_12192301.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_594920fed92cb3d2df458981c487dceb to: /var/lib/boinc/slots/50/ifsdata/ifsdata_12192301.zip
Unzipping the ifsdata_zip file: /var/lib/boinc/slots/50/ifsdata/ifsdata_12192301.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc/slots/50/159l_2/clim_data_12192301.zip
Unzipping the climate data zip file: /var/lib/boinc/slots/50/159l_2/clim_data_12192301.zip
Checking for progress XML file: /var/lib/boinc/slots/50/progress_file_12192301.xml
Opened progress file ok : /var/lib/boinc/slots/50/progress_file_12192301.xml
-- Model is restarting --
Adjusting last_iter, 2634, to previous model restart step.
Creating progress file: /var/lib/boinc/slots/50/progress_file_12192301.xml
last_cpu_time: 234053
upload_file_number: 109
last_iter: 2615
last_upload: 9417600
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0657_2006050100_123_975_12192301_0_r443492939
The child process has been launched with process id: 3270
Executing the command: /var/lib/boinc/slots/50/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:3270:3270] [20230126:033258:1674703978:0.002] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
03:40:18 STEP 2616 H=2616:00 +CPU= 74.100
03:42:52 STEP 2617 H=2617:00 +CPU= 34.447
03:45:31 STEP 2618 H=2618:00 +CPU= 33.887
03:49:26 STEP 2619 H=2619:00 +CPU= 50.948
03:52:58 STEP 2620 H=2620:00 +CPU= 39.590
03:55:27 STEP 2621 H=2621:00 +CPU= 33.758
03:59:34 STEP 2622 H=2622:00 +CPU= 51.566
04:01:44 STEP 2623 H=2623:00 +CPU= 29.743
04:04:13 STEP 2624 H=2624:00 +CPU= 31.017
04:08:15 STEP 2625 H=2625:00 +CPU= 50.163
04:11:29 STEP 2626 H=2626:00 +CPU= 36.202
04:13:42 STEP 2627 H=2627:00 +CPU= 30.258
04:18:47 STEP 2628 H=2628:00 +CPU= 61.620
Moving to projects directory: /var/lib/boinc/slots/50/ICMGGhq0f+002628
Moving to projects directory: /var/lib/boinc/slots/50/ICMSHhq0f+002628
Moving to projects directory: /var/lib/boinc/slots/50/ICMUAhq0f+002628
04:20:44 STEP 2629 H=2629:00 +CPU= 29.270
04:22:54 STEP 2630 H=2630:00 +CPU= 29.789
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2006050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0657
(argv4) batchid: 975
(argv5) wuid: 12192301
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc/slots/50
Project directory is: /var/lib/boinc/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301
..mkdir for temp folder for results failed
Copying: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc/slots/50/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc/slots/50/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_3f7806fb17fbdd54baa09aa36b4c4dc2 to: /var/lib/boinc/slots/50/oifs_43r3_ps_0657_2006050100_123_975_12192301.zip
Unzipping the namelist zip file: /var/lib/boinc/slots/50/oifs_43r3_ps_0657_2006050100_123_975_12192301.zip
ic_ancil_file: ic_ancil_12192301
ifsdata_file: ifsdata_12192301
climate_data_file: clim_data_12192301
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_271e89d9461af3fff859695ac7f7ded8 to: /var/lib/boinc/slots/50/ic_ancil_12192301.zip
Unzipping the IC ancils zip file: /var/lib/boinc/slots/50/ic_ancil_12192301.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_594920fed92cb3d2df458981c487dceb to: /var/lib/boinc/slots/50/ifsdata/ifsdata_12192301.zip
Unzipping the ifsdata_zip file: /var/lib/boinc/slots/50/ifsdata/ifsdata_12192301.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc/slots/50/159l_2/clim_data_12192301.zip
Unzipping the climate data zip file: /var/lib/boinc/slots/50/159l_2/clim_data_12192301.zip
Checking for progress XML file: /var/lib/boinc/slots/50/progress_file_12192301.xml
Opened progress file ok : /var/lib/boinc/slots/50/progress_file_12192301.xml
-- Model is restarting --
Adjusting last_iter, 2631, to previous model restart step.
Creating progress file: /var/lib/boinc/slots/50/progress_file_12192301.xml
last_cpu_time: 234716
upload_file_number: 109
last_iter: 2615
last_upload: 9417600
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0657_2006050100_123_975_12192301_0_r443492939
The child process has been launched with process id: 61
Executing the command: /var/lib/boinc/slots/50/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.002] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.002] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.002] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.002] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.002] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.002] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.003] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.003] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.003] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.003] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.004] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.004] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.004] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.004] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.005] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.005] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.005] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.006] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.006] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.006] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.007] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.007] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.007] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.007] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.008] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:61:61] [20230126:042949:1674707389:0.008] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
04:33:49 STEP 2616 H=2616:00 +CPU= 53.537
04:36:04 STEP 2617 H=2617:00 +CPU= 31.665
04:38:02 STEP 2618 H=2618:00 +CPU= 29.664
04:42:05 STEP 2619 H=2619:00 +CPU= 50.633
04:45:03 STEP 2620 H=2620:00 +CPU= 35.486
04:47:28 STEP 2621 H=2621:00 +CPU= 31.799
04:51:31 STEP 2622 H=2622:00 +CPU= 54.006
04:54:25 STEP 2623 H=2623:00 +CPU= 36.847
04:57:13 STEP 2624 H=2624:00 +CPU= 35.023
05:00:19 STEP 2625 H=2625:00 +CPU= 48.665
05:02:48 STEP 2626 H=2626:00 +CPU= 35.265
05:04:49 STEP 2627 H=2627:00 +CPU= 30.458
05:08:33 STEP 2628 H=2628:00 +CPU= 54.274
Moving to projects directory: /var/lib/boinc/slots/50/ICMGGhq0f+002628
Moving to projects directory: /var/lib/boinc/slots/50/ICMSHhq0f+002628
Moving to projects directory: /var/lib/boinc/slots/50/ICMUAhq0f+002628
05:10:36 STEP 2629 H=2629:00 +CPU= 31.491
05:12:50 STEP 2630 H=2630:00 +CPU= 33.299
05:15:13 STEP 2631 H=2631:00 +CPU= 41.718
05:17:11 STEP 2632 H=2632:00 +CPU= 29.866
05:19:21 STEP 2633 H=2633:00 +CPU= 32.634
05:22:29 STEP 2634 H=2634:00 +CPU= 48.048
05:24:16 STEP 2635 H=2635:00 +CPU= 28.438
05:26:18 STEP 2636 H=2636:00 +CPU= 31.781
05:28:55 STEP 2637 H=2637:00 +CPU= 42.711
05:31:35 STEP 2638 H=2638:00 +CPU= 34.473
05:36:06 STEP 2639 H=2639:00 +CPU= 47.071
Quit request received from BOINC client, ending the child process
..Failed, model did not complete successfully
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2006050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0657
(argv4) batchid: 975
(argv5) wuid: 12192301
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc/slots/50
Project directory is: /var/lib/boinc/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301
..mkdir for temp folder for results failed
Copying: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc/slots/50/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc/slots/50/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_3f7806fb17fbdd54baa09aa36b4c4dc2 to: /var/lib/boinc/slots/50/oifs_43r3_ps_0657_2006050100_123_975_12192301.zip
Unzipping the namelist zip file: /var/lib/boinc/slots/50/oifs_43r3_ps_0657_2006050100_123_975_12192301.zip
ic_ancil_file: ic_ancil_12192301
ifsdata_file: ifsdata_12192301
climate_data_file: clim_data_12192301
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_271e89d9461af3fff859695ac7f7ded8 to: /var/lib/boinc/slots/50/ic_ancil_12192301.zip
Unzipping the IC ancils zip file: /var/lib/boinc/slots/50/ic_ancil_12192301.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_594920fed92cb3d2df458981c487dceb to: /var/lib/boinc/slots/50/ifsdata/ifsdata_12192301.zip
Unzipping the ifsdata_zip file: /var/lib/boinc/slots/50/ifsdata/ifsdata_12192301.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc/slots/50/159l_2/clim_data_12192301.zip
Unzipping the climate data zip file: /var/lib/boinc/slots/50/159l_2/clim_data_12192301.zip
Checking for progress XML file: /var/lib/boinc/slots/50/progress_file_12192301.xml
Opened progress file ok : /var/lib/boinc/slots/50/progress_file_12192301.xml
-- Model is restarting --
Adjusting last_iter, 2640, to previous model restart step.
Creating progress file: /var/lib/boinc/slots/50/progress_file_12192301.xml
last_cpu_time: 235661
upload_file_number: 109
last_iter: 2639
last_upload: 9417600
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0657_2006050100_123_975_12192301_0_r443492939
The child process has been launched with process id: 766
Executing the command: /var/lib/boinc/slots/50/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:766:766] [20230126:053859:1674711539:0.001] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
05:44:11 STEP 2640 H=2640:00 +CPU= 67.273
Moving to projects directory: /var/lib/boinc/slots/50/ICMGGhq0f+002640
Moving to projects directory: /var/lib/boinc/slots/50/ICMSHhq0f+002640
Moving to projects directory: /var/lib/boinc/slots/50/ICMUAhq0f+002640
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMGGhq0f+002616
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMSHhq0f+002616
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMUAhq0f+002616
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMGGhq0f+002628
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMSHhq0f+002628
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMUAhq0f+002628
Zipping up the intermediate file: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_0657_2006050100_123_975_12192301_0_r443492939_109.zip
Uploading the intermediate file: upload_file_109.zip
05:46:10 STEP 2641 H=2641:00 +CPU= 31.850
05:48:24 STEP 2642 H=2642:00 +CPU= 35.975
05:53:13 STEP 2643 H=2643:00 +CPU= 60.539
05:55:49 STEP 2644 H=2644:00 +CPU= 34.877
05:59:03 STEP 2645 H=2645:00 +CPU= 37.344
06:02:43 STEP 2646 H=2646:00 +CPU= 52.722
06:05:58 STEP 2647 H=2647:00 +CPU= 36.672
06:07:59 STEP 2648 H=2648:00 +CPU= 31.003
06:11:36 STEP 2649 H=2649:00 +CPU= 49.921
06:14:00 STEP 2650 H=2650:00 +CPU= 35.012
06:16:13 STEP 2651 H=2651:00 +CPU= 32.841
06:19:50 STEP 2652 H=2652:00 +CPU= 55.012
Moving to projects directory: /var/lib/boinc/slots/50/ICMGGhq0f+002652
Moving to projects directory: /var/lib/boinc/slots/50/ICMSHhq0f+002652
Moving to projects directory: /var/lib/boinc/slots/50/ICMUAhq0f+002652
06:23:39 STEP 2653 H=2653:00 +CPU= 40.095
06:28:18 STEP 2654 H=2654:00 +CPU= 47.767
06:36:51 STEP 2655 H=2655:00 +CPU= 87.534
06:42:23 STEP 2656 H=2656:00 +CPU= 55.736
06:47:46 STEP 2657 H=2657:00 +CPU= 54.868
06:56:10 STEP 2658 H=2658:00 +CPU= 84.777
07:01:15 STEP 2659 H=2659:00 +CPU= 51.626
07:06:32 STEP 2660 H=2660:00 +CPU= 53.620
07:14:32 STEP 2661 H=2661:00 +CPU= 80.253
07:19:52 STEP 2662 H=2662:00 +CPU= 54.502
07:24:24 STEP 2663 H=2663:00 +CPU= 49.941
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2006050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0657
(argv4) batchid: 975
(argv5) wuid: 12192301
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc/slots/50
Project directory is: /var/lib/boinc/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301
..mkdir for temp folder for results failed
Copying: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc/slots/50/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc/slots/50/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_3f7806fb17fbdd54baa09aa36b4c4dc2 to: /var/lib/boinc/slots/50/oifs_43r3_ps_0657_2006050100_123_975_12192301.zip
Unzipping the namelist zip file: /var/lib/boinc/slots/50/oifs_43r3_ps_0657_2006050100_123_975_12192301.zip
ic_ancil_file: ic_ancil_12192301
ifsdata_file: ifsdata_12192301
climate_data_file: clim_data_12192301
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_271e89d9461af3fff859695ac7f7ded8 to: /var/lib/boinc/slots/50/ic_ancil_12192301.zip
Unzipping the IC ancils zip file: /var/lib/boinc/slots/50/ic_ancil_12192301.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_594920fed92cb3d2df458981c487dceb to: /var/lib/boinc/slots/50/ifsdata/ifsdata_12192301.zip
Unzipping the ifsdata_zip file: /var/lib/boinc/slots/50/ifsdata/ifsdata_12192301.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc/slots/50/159l_2/clim_data_12192301.zip
Unzipping the climate data zip file: /var/lib/boinc/slots/50/159l_2/clim_data_12192301.zip
Checking for progress XML file: /var/lib/boinc/slots/50/progress_file_12192301.xml
Opened progress file ok : /var/lib/boinc/slots/50/progress_file_12192301.xml
-- Model is restarting --
Adjusting last_iter, 2664, to previous model restart step.
Creating progress file: /var/lib/boinc/slots/50/progress_file_12192301.xml
last_cpu_time: 236910
upload_file_number: 110
last_iter: 2663
last_upload: 9504000
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0657_2006050100_123_975_12192301_0_r443492939
The child process has been launched with process id: 1230
Executing the command: /var/lib/boinc/slots/50/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.081] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.081] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.081] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.081] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.081] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.082] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.082] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.082] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.082] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.082] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.082] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.082] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.082] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.082] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.082] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.083] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.083] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.083] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.083] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.083] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.083] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.083] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.083] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.083] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.084] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.084] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.084] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.084] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.084] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1230:1230] [20230126:072819:1674718099:0.084] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
07:36:24 STEP 2664 H=2664:00 +CPU= 81.748
Moving to projects directory: /var/lib/boinc/slots/50/ICMGGhq0f+002664
Moving to projects directory: /var/lib/boinc/slots/50/ICMSHhq0f+002664
Moving to projects directory: /var/lib/boinc/slots/50/ICMUAhq0f+002664
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMGGhq0f+002640
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMSHhq0f+002640
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMUAhq0f+002640
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMGGhq0f+002652
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMSHhq0f+002652
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMUAhq0f+002652
Zipping up the intermediate file: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_0657_2006050100_123_975_12192301_0_r443492939_110.zip
Uploading the intermediate file: upload_file_110.zip
07:38:47 STEP 2665 H=2665:00 +CPU= 33.931
07:42:01 STEP 2666 H=2666:00 +CPU= 37.607
07:45:21 STEP 2667 H=2667:00 +CPU= 47.062
07:48:18 STEP 2668 H=2668:00 +CPU= 35.100
07:50:34 STEP 2669 H=2669:00 +CPU= 30.992
07:57:17 STEP 2670 H=2670:00 +CPU= 68.055
08:02:23 STEP 2671 H=2671:00 +CPU= 50.109
08:07:58 STEP 2672 H=2672:00 +CPU= 55.443
08:15:41 STEP 2673 H=2673:00 +CPU= 77.764
08:20:51 STEP 2674 H=2674:00 +CPU= 51.440
08:26:01 STEP 2675 H=2675:00 +CPU= 51.112
08:34:17 STEP 2676 H=2676:00 +CPU= 82.368
Moving to projects directory: /var/lib/boinc/slots/50/ICMGGhq0f+002676
Moving to projects directory: /var/lib/boinc/slots/50/ICMSHhq0f+002676
Moving to projects directory: /var/lib/boinc/slots/50/ICMUAhq0f+002676
08:39:25 STEP 2677 H=2677:00 +CPU= 51.375
08:44:44 STEP 2678 H=2678:00 +CPU= 52.816
08:52:03 STEP 2679 H=2679:00 +CPU= 72.746
08:57:15 STEP 2680 H=2680:00 +CPU= 52.460
09:02:10 STEP 2681 H=2681:00 +CPU= 50.122
09:09:39 STEP 2682 H=2682:00 +CPU= 75.861
09:14:36 STEP 2683 H=2683:00 +CPU= 49.253
09:19:49 STEP 2684 H=2684:00 +CPU= 52.821
09:27:51 STEP 2685 H=2685:00 +CPU= 80.855
09:33:23 STEP 2686 H=2686:00 +CPU= 56.351
09:39:15 STEP 2687 H=2687:00 +CPU= 59.139
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2006050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0657
(argv4) batchid: 975
(argv5) wuid: 12192301
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc/slots/50
Project directory is: /var/lib/boinc/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301
..mkdir for temp folder for results failed
Copying: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc/slots/50/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc/slots/50/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_3f7806fb17fbdd54baa09aa36b4c4dc2 to: /var/lib/boinc/slots/50/oifs_43r3_ps_0657_2006050100_123_975_12192301.zip
Unzipping the namelist zip file: /var/lib/boinc/slots/50/oifs_43r3_ps_0657_2006050100_123_975_12192301.zip
ic_ancil_file: ic_ancil_12192301
ifsdata_file: ifsdata_12192301
climate_data_file: clim_data_12192301
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_271e89d9461af3fff859695ac7f7ded8 to: /var/lib/boinc/slots/50/ic_ancil_12192301.zip
Unzipping the IC ancils zip file: /var/lib/boinc/slots/50/ic_ancil_12192301.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_594920fed92cb3d2df458981c487dceb to: /var/lib/boinc/slots/50/ifsdata/ifsdata_12192301.zip
Unzipping the ifsdata_zip file: /var/lib/boinc/slots/50/ifsdata/ifsdata_12192301.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc/slots/50/159l_2/clim_data_12192301.zip
Unzipping the climate data zip file: /var/lib/boinc/slots/50/159l_2/clim_data_12192301.zip
Checking for progress XML file: /var/lib/boinc/slots/50/progress_file_12192301.xml
Opened progress file ok : /var/lib/boinc/slots/50/progress_file_12192301.xml
-- Model is restarting --
Adjusting last_iter, 2688, to previous model restart step.
Creating progress file: /var/lib/boinc/slots/50/progress_file_12192301.xml
last_cpu_time: 238360
upload_file_number: 111
last_iter: 2687
last_upload: 9590400
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0657_2006050100_123_975_12192301_0_r443492939
The child process has been launched with process id: 1473
Executing the command: /var/lib/boinc/slots/50/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.001] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:dd2a2d7e4511:1:1:1473:1473] [20230126:095048:1674726648:0.002] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
09:57:06 STEP 2688 H=2688:00 +CPU= 74.608
Moving to projects directory: /var/lib/boinc/slots/50/ICMGGhq0f+002688
Moving to projects directory: /var/lib/boinc/slots/50/ICMSHhq0f+002688
Moving to projects directory: /var/lib/boinc/slots/50/ICMUAhq0f+002688
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMGGhq0f+002664
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMSHhq0f+002664
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMUAhq0f+002664
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMGGhq0f+002676
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMSHhq0f+002676
Adding to the zip: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_12192301/ICMUAhq0f+002676
Zipping up the intermediate file: /var/lib/boinc/projects/climateprediction.net/oifs_43r3_ps_0657_2006050100_123_975_12192301_0_r443492939_111.zip
Uploading the intermediate file: upload_file_111.zip
09:59:08 STEP 2689 H=2689:00 +CPU= 31.626
10:01:27 STEP 2690 H=2690:00 +CPU= 34.180
10:04:04 STEP 2691 H=2691:00 +CPU= 45.066
10:06:19 STEP 2692 H=2692:00 +CPU= 32.006
10:10:26 STEP 2693 H=2693:00 +CPU= 43.386
10:19:09 STEP 2694 H=2694:00 +CPU= 90.469
10:25:03 STEP 2695 H=2695:00 +CPU= 61.506
10:30:43 STEP 2696 H=2696:00 +CPU= 59.638
10:40:07 STEP 2697 H=2697:00 +CPU= 97.701
10:46:09 STEP 2698 H=2698:00 +CPU= 62.181
10:52:35 STEP 2699 H=2699:00 +CPU= 65.261
11:02:46 STEP 2700 H=2700:00 +CPU=104.218
Moving to projects directory: /var/lib/boinc/slots/50/ICMGGhq0f+002700
Moving to projects directory: /var/lib/boinc/slots/50/ICMSHhq0f+002700
Moving to projects directory: /var/lib/boinc/slots/50/ICMUAhq0f+002700
11:08:56 STEP 2701 H=2701:00 +CPU= 64.179
11:14:49 STEP 2702 H=2702:00 +CPU= 60.933
11:23:31 STEP 2703 H=2703:00 +CPU= 89.972
11:29:16 STEP 2704 H=2704:00 +CPU= 60.304
11:34:48 STEP 2705 H=2705:00 +CPU= 57.383
11:43:56 STEP 2706 H=2706:00 +CPU= 94.705
11:48:54 STEP 2707 H=2707:00 +CPU= 55.907
11:53:44 STEP 2708 H=2708:00 +CPU= 52.363
12:02:10 STEP 2709 H=2709:00 +CPU= 86.916
12:08:03 STEP 2710 H=2710:00 +CPU= 61.134
12:14:22 STEP 2711 H=2711:00 +CPU= 65.777
Abort request received from BOINC client, ending the child process
..Failed, model did not complete successfully
</stderr_txt>
]]>
