<core_client_version>7.16.6</core_client_version>
<![CDATA[
<stderr_txt>
713:1677546433:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:12494:12494] [20230227:170713:1677546433:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:12494:12494] [20230227:170713:1677546433:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:12494:12494] [20230227:170713:1677546433:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:12494:12494] [20230227:170713:1677546433:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:12494:12494] [20230227:170713:1677546433:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:12494:12494] [20230227:170713:1677546433:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:12494:12494] [20230227:170713:1677546433:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:12494:12494] [20230227:170713:1677546433:0.000] [catch_signals@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
17:07:53 STEP 2784 H=2784:00 +CPU= 38.267
17:08:15 STEP 2785 H=2785:00 +CPU= 21.170
17:08:37 STEP 2786 H=2786:00 +CPU= 21.832
17:09:11 STEP 2787 H=2787:00 +CPU= 34.164
17:09:33 STEP 2788 H=2788:00 +CPU= 21.854
17:09:54 STEP 2789 H=2789:00 +CPU= 21.205
17:10:29 STEP 2790 H=2790:00 +CPU= 34.965
17:10:50 STEP 2791 H=2791:00 +CPU= 21.255
17:11:12 STEP 2792 H=2792:00 +CPU= 21.941
17:11:46 STEP 2793 H=2793:00 +CPU= 34.273
17:12:08 STEP 2794 H=2794:00 +CPU= 21.932
17:12:30 STEP 2795 H=2795:00 +CPU= 21.265
17:13:07 STEP 2796 H=2796:00 +CPU= 37.160
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGhnw4+002796
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHhnw4+002796
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAhnw4+002796
17:13:28 STEP 2797 H=2797:00 +CPU= 21.287
17:13:50 STEP 2798 H=2798:00 +CPU= 21.945
17:14:25 STEP 2799 H=2799:00 +CPU= 34.272
17:14:46 STEP 2800 H=2800:00 +CPU= 21.923
17:15:08 STEP 2801 H=2801:00 +CPU= 21.256
17:15:43 STEP 2802 H=2802:00 +CPU= 34.988
17:16:04 STEP 2803 H=2803:00 +CPU= 21.251
17:16:26 STEP 2804 H=2804:00 +CPU= 21.902
17:17:00 STEP 2805 H=2805:00 +CPU= 34.215
17:17:22 STEP 2806 H=2806:00 +CPU= 21.903
17:17:44 STEP 2807 H=2807:00 +CPU= 21.840
17:18:23 STEP 2808 H=2808:00 +CPU= 39.243
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGhnw4+002808
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHhnw4+002808
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAhnw4+002808
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMGGhnw4+002784
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMSHhnw4+002784
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMUAhnw4+002784
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMGGhnw4+002796
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMSHhnw4+002796
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMUAhnw4+002796
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_00md_2019110100_123_993_12214243_0_r1117814735_116.zip
Uploading the intermediate file: upload_file_116.zip
17:18:45 STEP 2809 H=2809:00 +CPU= 21.209
17:19:07 STEP 2810 H=2810:00 +CPU= 21.875
17:19:41 STEP 2811 H=2811:00 +CPU= 34.236
17:20:03 STEP 2812 H=2812:00 +CPU= 21.940
17:20:24 STEP 2813 H=2813:00 +CPU= 21.288
17:20:59 STEP 2814 H=2814:00 +CPU= 35.115
17:21:21 STEP 2815 H=2815:00 +CPU= 21.318
17:21:43 STEP 2816 H=2816:00 +CPU= 21.986
17:22:17 STEP 2817 H=2817:00 +CPU= 34.371
17:22:39 STEP 2818 H=2818:00 +CPU= 21.980
17:23:00 STEP 2819 H=2819:00 +CPU= 21.300
17:23:38 STEP 2820 H=2820:00 +CPU= 37.275
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGhnw4+002820
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHhnw4+002820
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAhnw4+002820
17:23:59 STEP 2821 H=2821:00 +CPU= 21.313
17:24:21 STEP 2822 H=2822:00 +CPU= 21.958
17:24:55 STEP 2823 H=2823:00 +CPU= 34.341
17:25:17 STEP 2824 H=2824:00 +CPU= 21.986
17:25:39 STEP 2825 H=2825:00 +CPU= 21.278
17:26:14 STEP 2826 H=2826:00 +CPU= 35.024
17:26:35 STEP 2827 H=2827:00 +CPU= 21.270
17:26:57 STEP 2828 H=2828:00 +CPU= 21.935
17:27:31 STEP 2829 H=2829:00 +CPU= 34.271
17:27:53 STEP 2830 H=2830:00 +CPU= 21.885
17:28:15 STEP 2831 H=2831:00 +CPU= 21.823
17:28:55 STEP 2832 H=2832:00 +CPU= 39.311
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGhnw4+002832
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHhnw4+002832
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAhnw4+002832
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMGGhnw4+002808
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMSHhnw4+002808
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMUAhnw4+002808
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMGGhnw4+002820
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMSHhnw4+002820
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMUAhnw4+002820
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_00md_2019110100_123_993_12214243_0_r1117814735_117.zip
Uploading the intermediate file: upload_file_117.zip
17:29:16 STEP 2833 H=2833:00 +CPU= 21.240
17:29:38 STEP 2834 H=2834:00 +CPU= 21.905
17:30:13 STEP 2835 H=2835:00 +CPU= 34.305
17:30:34 STEP 2836 H=2836:00 +CPU= 21.901
17:30:56 STEP 2837 H=2837:00 +CPU= 21.281
17:31:31 STEP 2838 H=2838:00 +CPU= 35.102
17:31:52 STEP 2839 H=2839:00 +CPU= 21.303
17:32:14 STEP 2840 H=2840:00 +CPU= 21.966
17:32:49 STEP 2841 H=2841:00 +CPU= 34.359
17:33:11 STEP 2842 H=2842:00 +CPU= 21.990
17:33:32 STEP 2843 H=2843:00 +CPU= 21.312
17:34:09 STEP 2844 H=2844:00 +CPU= 37.321
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGhnw4+002844
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHhnw4+002844
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAhnw4+002844
17:34:31 STEP 2845 H=2845:00 +CPU= 21.308
17:34:53 STEP 2846 H=2846:00 +CPU= 21.983
17:35:27 STEP 2847 H=2847:00 +CPU= 34.285
17:35:49 STEP 2848 H=2848:00 +CPU= 21.923
17:36:10 STEP 2849 H=2849:00 +CPU= 21.249
17:36:45 STEP 2850 H=2850:00 +CPU= 35.006
17:37:06 STEP 2851 H=2851:00 +CPU= 21.231
17:37:28 STEP 2852 H=2852:00 +CPU= 21.892
17:38:03 STEP 2853 H=2853:00 +CPU= 34.228
17:38:25 STEP 2854 H=2854:00 +CPU= 21.870
17:38:46 STEP 2855 H=2855:00 +CPU= 21.808
17:39:26 STEP 2856 H=2856:00 +CPU= 39.243
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGhnw4+002856
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHhnw4+002856
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAhnw4+002856
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMGGhnw4+002832
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMSHhnw4+002832
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMUAhnw4+002832
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMGGhnw4+002844
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMSHhnw4+002844
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMUAhnw4+002844
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_00md_2019110100_123_993_12214243_0_r1117814735_118.zip
Uploading the intermediate file: upload_file_118.zip
17:39:47 STEP 2857 H=2857:00 +CPU= 21.189
17:40:09 STEP 2858 H=2858:00 +CPU= 21.739
17:40:43 STEP 2859 H=2859:00 +CPU= 33.998
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_1.21_x86_64-pc-linux-gnu
(argv1) start_date: 2019110100
(argv2) exptid: hnw4
(argv3) unique_member_id: 00md
(argv4) batchid: 993
(argv5) wuid: 12214243
(argv6) fclen: 123
(argv7) app_name: oifs_43r3
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/0
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3
version: 1.21
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_app_1.21_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/0/oifs_43r3_app_1.21_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/0/oifs_43r3_app_1.21_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_beed77cb7ca04912f21f9d910a0f695f to: /var/lib/boinc-client/slots/0/oifs_43r3_00md_2019110100_123_993_12214243.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/0/oifs_43r3_00md_2019110100_123_993_12214243.zip
ic_ancil_file: ic_ancil_12214243
ifsdata_file: ifsdata_12214243
climate_data_file: clim_data_12214243
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_4afc5af9d2fa21fe864c5f3a0175526b to: /var/lib/boinc-client/slots/0/ic_ancil_12214243.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/0/ic_ancil_12214243.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_623e64a92c1c312c0962bbb8329b92a2 to: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12214243.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12214243.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/0/159l_2/clim_data_12214243.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/0/159l_2/clim_data_12214243.zip
Checking for progress XML file: /var/lib/boinc-client/slots/0/progress_file_12214243.xml
Opened progress file ok : /var/lib/boinc-client/slots/0/progress_file_12214243.xml
-- Model is restarting --
Adjusting last_iter, 2860, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/0/progress_file_12214243.xml
last_cpu_time: 78126
upload_file_number: 119
last_iter: 2857
last_upload: 10281600
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_00md_2019110100_123_993_12214243_0_r1117814735
The child process has been launched with process id: 2556
Executing the command: /var/lib/boinc-client/slots/0/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2091] fp = 0x3124020
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [ignore_signals@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [restore_default_signals@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [trapfpe_treatment@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:2556:2556] [20230228:024700:1677581220:0.000] [catch_signals@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
02:47:48 STEP 2856 H=2856:00 +CPU= 38.391
02:48:09 STEP 2857 H=2857:00 +CPU= 21.175
02:48:31 STEP 2858 H=2858:00 +CPU= 21.843
02:49:05 STEP 2859 H=2859:00 +CPU= 34.211
02:49:27 STEP 2860 H=2860:00 +CPU= 21.866
02:49:48 STEP 2861 H=2861:00 +CPU= 21.221
02:50:23 STEP 2862 H=2862:00 +CPU= 35.016
02:50:44 STEP 2863 H=2863:00 +CPU= 21.266
02:51:06 STEP 2864 H=2864:00 +CPU= 21.929
02:51:41 STEP 2865 H=2865:00 +CPU= 34.300
02:52:03 STEP 2866 H=2866:00 +CPU= 21.922
02:52:24 STEP 2867 H=2867:00 +CPU= 21.268
02:53:01 STEP 2868 H=2868:00 +CPU= 37.194
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGhnw4+002868
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHhnw4+002868
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAhnw4+002868
02:53:22 STEP 2869 H=2869:00 +CPU= 21.235
(argv0) ../../projects/climateprediction.net/oifs_43r3_1.21_x86_64-pc-linux-gnu
(argv1) start_date: 2019110100
(argv2) exptid: hnw4
(argv3) unique_member_id: 00md
(argv4) batchid: 993
(argv5) wuid: 12214243
(argv6) fclen: 123
(argv7) app_name: oifs_43r3
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/0
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3
version: 1.21
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_app_1.21_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/0/oifs_43r3_app_1.21_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/0/oifs_43r3_app_1.21_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_beed77cb7ca04912f21f9d910a0f695f to: /var/lib/boinc-client/slots/0/oifs_43r3_00md_2019110100_123_993_12214243.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/0/oifs_43r3_00md_2019110100_123_993_12214243.zip
ic_ancil_file: ic_ancil_12214243
ifsdata_file: ifsdata_12214243
climate_data_file: clim_data_12214243
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_4afc5af9d2fa21fe864c5f3a0175526b to: /var/lib/boinc-client/slots/0/ic_ancil_12214243.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/0/ic_ancil_12214243.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_623e64a92c1c312c0962bbb8329b92a2 to: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12214243.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/0/ifsdata/ifsdata_12214243.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/0/159l_2/clim_data_12214243.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/0/159l_2/clim_data_12214243.zip
Checking for progress XML file: /var/lib/boinc-client/slots/0/progress_file_12214243.xml
Opened progress file ok : /var/lib/boinc-client/slots/0/progress_file_12214243.xml
-- Model is restarting --
Adjusting last_iter, 2870, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/0/progress_file_12214243.xml
last_cpu_time: 78509
upload_file_number: 119
last_iter: 2857
last_upload: 10281600
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_00md_2019110100_123_993_12214243_0_r1117814735
The child process has been launched with process id: 1567
Executing the command: /var/lib/boinc-client/slots/0/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2091] fp = 0x3124020
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [process_options@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [ignore_signals@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [restore_default_signals@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [trapfpe_treatment@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x810730 (old at 0x1ce8440)
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [signal_drhook_init@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x810730 (old at (nil))
[EC_DRHOOK:CB-BOINC-10:1:1:1567:1567] [20230228:025554:1677581754:0.000] [catch_signals@/home/glenn/github/gc_oifs43r3/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
02:56:55 STEP 2856 H=2856:00 +CPU= 38.393
02:57:18 STEP 2857 H=2857:00 +CPU= 21.237
02:57:40 STEP 2858 H=2858:00 +CPU= 21.992
02:58:15 STEP 2859 H=2859:00 +CPU= 34.316
02:58:37 STEP 2860 H=2860:00 +CPU= 21.934
02:58:58 STEP 2861 H=2861:00 +CPU= 21.269
02:59:33 STEP 2862 H=2862:00 +CPU= 35.063
02:59:55 STEP 2863 H=2863:00 +CPU= 21.284
03:00:17 STEP 2864 H=2864:00 +CPU= 21.981
03:00:51 STEP 2865 H=2865:00 +CPU= 34.315
03:01:13 STEP 2866 H=2866:00 +CPU= 21.931
03:01:35 STEP 2867 H=2867:00 +CPU= 21.280
03:02:12 STEP 2868 H=2868:00 +CPU= 37.248
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGhnw4+002868
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHhnw4+002868
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAhnw4+002868
03:02:33 STEP 2869 H=2869:00 +CPU= 21.298
03:02:55 STEP 2870 H=2870:00 +CPU= 21.978
03:03:30 STEP 2871 H=2871:00 +CPU= 34.275
03:03:52 STEP 2872 H=2872:00 +CPU= 21.954
03:04:13 STEP 2873 H=2873:00 +CPU= 21.278
03:04:48 STEP 2874 H=2874:00 +CPU= 35.017
03:05:09 STEP 2875 H=2875:00 +CPU= 21.253
03:05:31 STEP 2876 H=2876:00 +CPU= 21.891
03:06:05 STEP 2877 H=2877:00 +CPU= 34.192
03:06:27 STEP 2878 H=2878:00 +CPU= 21.897
03:06:49 STEP 2879 H=2879:00 +CPU= 21.846
03:07:29 STEP 2880 H=2880:00 +CPU= 39.229
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGhnw4+002880
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHhnw4+002880
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAhnw4+002880
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMGGhnw4+002856
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMSHhnw4+002856
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMUAhnw4+002856
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMGGhnw4+002868
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMSHhnw4+002868
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_12214243/ICMUAhnw4+002868
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_00md_2019110100_123_993_12214243_0_r1117814735_119.zip
Uploading the intermediate file: upload_file_119.zip
03:07:50 STEP 2881 H=2881:00 +CPU= 21.235
03:08:12 STEP 2882 H=2882:00 +CPU= 21.927
03:08:46 STEP 2883 H=2883:00 +CPU= 34.273
03:09:08 STEP 2884 H=2884:00 +CPU= 21.935
03:09:30 STEP 2885 H=2885:00 +CPU= 21.294
03:10:05 STEP 2886 H=2886:00 +CPU= 35.064
03:10:26 STEP 2887 H=2887:00 +CPU= 21.318
03:10:48 STEP 2888 H=2888:00 +CPU= 21.983
03:11:22 STEP 2889 H=2889:00 +CPU= 34.317
03:11:44 STEP 2890 H=2890:00 +CPU= 21.983
03:12:06 STEP 2891 H=2891:00 +CPU= 21.319
03:12:43 STEP 2892 H=2892:00 +CPU= 37.245
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMGGhnw4+002892
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMSHhnw4+002892
Moving to projects directory: /var/lib/boinc-client/slots/0/ICMUAhnw4+002892
03:13:04 STEP 2893 H=2893:00 +CPU= 21.299
03:13:26 STEP 2894 H=2894:00 +CPU= 21.975
03:14:01 STEP 2895 H=2895:00 +CPU= 34.284
03:14:23 STEP 2896 H=2896:00 +CPU= 21.950
03:14:44 STEP 2897 H=2897:00 +CPU= 21.275
03:15:19 STEP 2898 H=2898:00 +CPU= 35.036
03:15:40 STEP 2899 H=2899:00 +CPU= 21.278
</stderr_txt>
]]>
