<core_client_version>7.18.1</core_client_version>
<![CDATA[
<message>
Disk usage limit exceeded</message>
<stderr_txt>
boinc-client/slots/4/ICMUAhq0f+001860
02:27:03 STEP 1861 H=1861:00 +CPU= 19.587
02:27:24 STEP 1862 H=1862:00 +CPU= 19.992
02:27:55 STEP 1863 H=1863:00 +CPU= 30.150
02:28:16 STEP 1864 H=1864:00 +CPU= 19.763
02:28:36 STEP 1865 H=1865:00 +CPU= 19.219
02:29:08 STEP 1866 H=1866:00 +CPU= 30.975
02:29:27 STEP 1867 H=1867:00 +CPU= 19.078
02:29:48 STEP 1868 H=1868:00 +CPU= 19.875
02:30:18 STEP 1869 H=1869:00 +CPU= 29.207
02:30:37 STEP 1870 H=1870:00 +CPU= 19.315
02:30:57 STEP 1871 H=1871:00 +CPU= 19.522
02:31:34 STEP 1872 H=1872:00 +CPU= 35.615
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMGGhq0f+001872
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMSHhq0f+001872
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMUAhq0f+001872
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001848
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001848
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001848
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001860
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001860
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001860
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0820_2018050100_123_987_12204464_0_r973255285_77.zip
Uploading the intermediate file: upload_file_77.zip
02:31:54 STEP 1873 H=1873:00 +CPU= 19.498
02:32:15 STEP 1874 H=1874:00 +CPU= 20.017
02:32:47 STEP 1875 H=1875:00 +CPU= 30.477
02:33:07 STEP 1876 H=1876:00 +CPU= 19.705
02:33:27 STEP 1877 H=1877:00 +CPU= 19.228
02:33:58 STEP 1878 H=1878:00 +CPU= 30.398
02:34:17 STEP 1879 H=1879:00 +CPU= 19.138
02:34:38 STEP 1880 H=1880:00 +CPU= 19.776
02:35:09 STEP 1881 H=1881:00 +CPU= 30.241
02:35:30 STEP 1882 H=1882:00 +CPU= 19.828
02:35:50 STEP 1883 H=1883:00 +CPU= 19.628
02:36:24 STEP 1884 H=1884:00 +CPU= 33.304
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMGGhq0f+001884
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMSHhq0f+001884
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMUAhq0f+001884
02:36:44 STEP 1885 H=1885:00 +CPU= 19.117
02:37:05 STEP 1886 H=1886:00 +CPU= 20.055
02:37:37 STEP 1887 H=1887:00 +CPU= 30.679
02:37:58 STEP 1888 H=1888:00 +CPU= 20.198
02:38:18 STEP 1889 H=1889:00 +CPU= 19.760
02:38:50 STEP 1890 H=1890:00 +CPU= 31.007
02:39:10 STEP 1891 H=1891:00 +CPU= 19.314
02:39:31 STEP 1892 H=1892:00 +CPU= 20.406
02:40:03 STEP 1893 H=1893:00 +CPU= 30.405
02:40:24 STEP 1894 H=1894:00 +CPU= 20.002
02:40:44 STEP 1895 H=1895:00 +CPU= 20.164
02:41:22 STEP 1896 H=1896:00 +CPU= 36.323
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMGGhq0f+001896
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMSHhq0f+001896
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMUAhq0f+001896
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001872
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001872
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001872
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001884
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001884
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001884
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0820_2018050100_123_987_12204464_0_r973255285_78.zip
Uploading the intermediate file: upload_file_78.zip
02:41:42 STEP 1897 H=1897:00 +CPU= 18.880
02:42:02 STEP 1898 H=1898:00 +CPU= 20.024
02:42:33 STEP 1899 H=1899:00 +CPU= 30.226
02:42:53 STEP 1900 H=1900:00 +CPU= 18.943
02:43:12 STEP 1901 H=1901:00 +CPU= 18.856
02:43:43 STEP 1902 H=1902:00 +CPU= 30.470
02:44:02 STEP 1903 H=1903:00 +CPU= 18.708
02:44:22 STEP 1904 H=1904:00 +CPU= 19.004
02:44:52 STEP 1905 H=1905:00 +CPU= 29.828
02:45:12 STEP 1906 H=1906:00 +CPU= 20.049
02:45:33 STEP 1907 H=1907:00 +CPU= 19.433
02:46:07 STEP 1908 H=1908:00 +CPU= 33.402
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMGGhq0f+001908
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMSHhq0f+001908
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMUAhq0f+001908
02:46:26 STEP 1909 H=1909:00 +CPU= 18.957
02:46:46 STEP 1910 H=1910:00 +CPU= 19.893
02:47:18 STEP 1911 H=1911:00 +CPU= 30.590
02:47:38 STEP 1912 H=1912:00 +CPU= 19.422
02:47:57 STEP 1913 H=1913:00 +CPU= 18.852
02:48:29 STEP 1914 H=1914:00 +CPU= 30.602
02:48:49 STEP 1915 H=1915:00 +CPU= 19.580
02:49:10 STEP 1916 H=1916:00 +CPU= 20.332
02:49:41 STEP 1917 H=1917:00 +CPU= 30.380
02:50:02 STEP 1918 H=1918:00 +CPU= 19.671
02:50:22 STEP 1919 H=1919:00 +CPU= 20.078
02:50:59 STEP 1920 H=1920:00 +CPU= 35.625
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMGGhq0f+001920
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMSHhq0f+001920
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMUAhq0f+001920
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001896
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001896
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001896
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001908
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001908
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001908
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0820_2018050100_123_987_12204464_0_r973255285_79.zip
Uploading the intermediate file: upload_file_79.zip
02:51:20 STEP 1921 H=1921:00 +CPU= 20.255
02:51:42 STEP 1922 H=1922:00 +CPU= 21.120
02:52:14 STEP 1923 H=1923:00 +CPU= 30.394
02:52:34 STEP 1924 H=1924:00 +CPU= 19.605
02:52:54 STEP 1925 H=1925:00 +CPU= 19.443
02:53:26 STEP 1926 H=1926:00 +CPU= 31.031
02:53:46 STEP 1927 H=1927:00 +CPU= 19.387
02:54:07 STEP 1928 H=1928:00 +CPU= 20.087
02:54:38 STEP 1929 H=1929:00 +CPU= 30.287
02:54:58 STEP 1930 H=1930:00 +CPU= 19.880
02:55:18 STEP 1931 H=1931:00 +CPU= 19.206
02:55:53 STEP 1932 H=1932:00 +CPU= 33.736
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMGGhq0f+001932
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMSHhq0f+001932
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMUAhq0f+001932
02:56:13 STEP 1933 H=1933:00 +CPU= 19.363
02:56:34 STEP 1934 H=1934:00 +CPU= 20.416
02:57:04 STEP 1935 H=1935:00 +CPU= 29.545
02:57:25 STEP 1936 H=1936:00 +CPU= 19.498
02:57:45 STEP 1937 H=1937:00 +CPU= 19.805
02:58:18 STEP 1938 H=1938:00 +CPU= 31.262
02:58:38 STEP 1939 H=1939:00 +CPU= 19.384
02:58:59 STEP 1940 H=1940:00 +CPU= 20.159
02:59:30 STEP 1941 H=1941:00 +CPU= 30.393
02:59:52 STEP 1942 H=1942:00 +CPU= 20.353
03:00:14 STEP 1943 H=1943:00 +CPU= 20.841
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
03:01:05 STEP 1944 H=1944:00 +CPU= 36.093
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMGGhq0f+001944
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMSHhq0f+001944
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMUAhq0f+001944
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001920
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001920
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001920
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001932
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001932
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001932
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0820_2018050100_123_987_12204464_0_r973255285_80.zip
Uploading the intermediate file: upload_file_80.zip
03:01:26 STEP 1945 H=1945:00 +CPU= 19.656
03:01:47 STEP 1946 H=1946:00 +CPU= 20.364
03:02:17 STEP 1947 H=1947:00 +CPU= 29.662
03:02:38 STEP 1948 H=1948:00 +CPU= 19.682
03:02:57 STEP 1949 H=1949:00 +CPU= 19.167
03:03:29 STEP 1950 H=1950:00 +CPU= 30.657
03:03:47 STEP 1951 H=1951:00 +CPU= 18.487
03:04:08 STEP 1952 H=1952:00 +CPU= 19.986
03:04:38 STEP 1953 H=1953:00 +CPU= 29.345
03:04:58 STEP 1954 H=1954:00 +CPU= 19.665
03:05:18 STEP 1955 H=1955:00 +CPU= 19.532
03:05:52 STEP 1956 H=1956:00 +CPU= 33.403
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMGGhq0f+001956
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMSHhq0f+001956
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMUAhq0f+001956
03:06:12 STEP 1957 H=1957:00 +CPU= 19.330
03:06:32 STEP 1958 H=1958:00 +CPU= 19.632
03:07:03 STEP 1959 H=1959:00 +CPU= 30.486
03:07:23 STEP 1960 H=1960:00 +CPU= 19.958
03:07:43 STEP 1961 H=1961:00 +CPU= 19.543
03:08:15 STEP 1962 H=1962:00 +CPU= 30.789
03:08:35 STEP 1963 H=1963:00 +CPU= 19.184
03:08:55 STEP 1964 H=1964:00 +CPU= 19.947
03:09:26 STEP 1965 H=1965:00 +CPU= 29.652
03:09:47 STEP 1966 H=1966:00 +CPU= 19.213
03:10:08 STEP 1967 H=1967:00 +CPU= 19.905
03:10:44 STEP 1968 H=1968:00 +CPU= 34.851
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMGGhq0f+001968
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMSHhq0f+001968
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMUAhq0f+001968
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001944
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001944
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001944
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001956
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001956
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001956
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0820_2018050100_123_987_12204464_0_r973255285_81.zip
Uploading the intermediate file: upload_file_81.zip
03:11:04 STEP 1969 H=1969:00 +CPU= 19.324
Uploading trickle at timestep: 7084800
03:11:24 STEP 1970 H=1970:00 +CPU= 19.776
03:11:55 STEP 1971 H=1971:00 +CPU= 29.950
03:12:15 STEP 1972 H=1972:00 +CPU= 19.654
03:12:35 STEP 1973 H=1973:00 +CPU= 19.202
03:13:07 STEP 1974 H=1974:00 +CPU= 30.435
03:13:27 STEP 1975 H=1975:00 +CPU= 19.213
03:13:47 STEP 1976 H=1976:00 +CPU= 19.867
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.732] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1538] Received signal#15 (SIGTERM) :: 3996MB (heap), 4419MB (maxrss), 0MB (maxstack), 0 (paging), nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.733] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1542] Also activating Harakiri-alarm (SIGALRM=14) to expire after 500s elapsed to prevent hangs, nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.733] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1544] Harakiri signal handler 'signal_harakiri' for signal#14 (SIGALRM) installed at 0x81f3c0 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.733] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1617] Signal#15 was caused by unrecognized si_code [memaddr=0x1], nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.733] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1686] Starting DrHook backtrace for signal#15, nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.733] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3843] 3996 MB (maxheap), 4419 MB (maxrss), 0 MB (maxstack)
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.733] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : MASTER
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.738] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT0
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.741] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT1
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT2
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT3
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT4
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : STEPO
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : SCAN2M
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : GP_MODEL_HEAP
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : GP_MODEL
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : EC_PHYS_DRV
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : >OMP-PHYSICS CLDPP T/S (1002)
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : EC_PHYS
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CALLPAR
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CONVECTION_LAYER
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CUCALLN
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CUMASTRN
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031409:1673576049:16439.742] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CUDUDV
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031410:1673576050:16440.742] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1734] DrHook backtrace done for signal#15, nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2926:2926] [20230113:031410:1673576050:16440.742] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1785] Calling previous signal handler at 0x1ce8bb0 for signal#15, nsigs = 1
forrtl: error (78): process killed (SIGTERM)
Image PC Routine Line Source
oifs_43r3_model.e 0000000001CE916B Unknown Unknown Unknown
oifs_43r3_model.e 000000000081FFF1 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DC9090 Unknown Unknown Unknown
oifs_43r3_model.e 00000000010F1654 cududv_ 262 cududv.F90
oifs_43r3_model.e 00000000010D658F cumastrn_ 1220 cumastrn.F90
oifs_43r3_model.e 00000000010C7792 cucalln_ 352 cucalln.F90
oifs_43r3_model.e 00000000010C4C15 convection_layer_ 121 convection_layer.F90
oifs_43r3_model.e 0000000000F11F5B callpar_ 1064 callpar.F90
oifs_43r3_model.e 0000000000E4940F ec_phys_ 670 ec_phys.F90
oifs_43r3_model.e 0000000000E3610C ec_phys_drv_ 599 ec_phys_drv.F90
oifs_43r3_model.e 0000000000BFA3D3 gp_model_ 613 gp_model.F90
oifs_43r3_model.e 00000000012E9482 gp_model_heap_ 74 gp_model_heap.F90
oifs_43r3_model.e 0000000000BBC1CE scan2m_ 535 scan2m.F90
oifs_43r3_model.e 000000000057A664 stepo_ 327 stepo.F90
oifs_43r3_model.e 000000000055DEEC cnt4_ 1133 cnt4.F90
oifs_43r3_model.e 00000000005471C9 cnt3_ 267 cnt3.F90
oifs_43r3_model.e 0000000000546412 cnt2_ 88 cnt2.F90
oifs_43r3_model.e 0000000000545E28 cnt1_ 92 cnt1.F90
oifs_43r3_model.e 000000000040708D cnt0_ 146 cnt0.F90
oifs_43r3_model.e 000000000040220F MAIN__ 96 master.F90
oifs_43r3_model.e 00000000004021A2 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DCA390 Unknown Unknown Unknown
oifs_43r3_model.e 000000000040206E Unknown Unknown Unknown
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2018050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0820
(argv4) batchid: 987
(argv5) wuid: 12204464
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/4
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/4/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/4/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_467d28a54f765caa463421df45eb7ad9 to: /var/lib/boinc-client/slots/4/oifs_43r3_ps_0820_2018050100_123_987_12204464.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/4/oifs_43r3_ps_0820_2018050100_123_987_12204464.zip
ic_ancil_file: ic_ancil_12204464
ifsdata_file: ifsdata_12204464
climate_data_file: clim_data_12204464
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_8ead1d3ea42fb9993f857c72ad5b7eac to: /var/lib/boinc-client/slots/4/ic_ancil_12204464.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/4/ic_ancil_12204464.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_61800d0db66ca28702ed0afc14a0c793 to: /var/lib/boinc-client/slots/4/ifsdata/ifsdata_12204464.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/4/ifsdata/ifsdata_12204464.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/4/159l_2/clim_data_12204464.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/4/159l_2/clim_data_12204464.zip
Checking for progress XML file: /var/lib/boinc-client/slots/4/progress_file_12204464.xml
Opened progress file ok : /var/lib/boinc-client/slots/4/progress_file_12204464.xml
-- Model is restarting --
Adjusting last_iter, 1977, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/4/progress_file_12204464.xml
last_cpu_time: 53204
upload_file_number: 82
last_iter: 1967
last_upload: 7084800
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0820_2018050100_123_987_12204464_0_r973255285
The child process has been launched with process id: 2953
Executing the command: /var/lib/boinc-client/slots/4/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212159:1674591719:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
21:22:43 STEP 1968 H=1968:00 +CPU= 29.822
21:22:58 STEP 1969 H=1969:00 +CPU= 14.698
21:23:14 STEP 1970 H=1970:00 +CPU= 15.501
21:23:40 STEP 1971 H=1971:00 +CPU= 24.304
21:23:57 STEP 1972 H=1972:00 +CPU= 15.727
21:24:13 STEP 1973 H=1973:00 +CPU= 15.574
21:24:38 STEP 1974 H=1974:00 +CPU= 24.713
21:24:54 STEP 1975 H=1975:00 +CPU= 16.237
21:25:08 STEP 1976 H=1976:00 +CPU= 14.289
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1538] Received signal#15 (SIGTERM) :: 4032MB (heap), 4403MB (maxrss), 0MB (maxstack), 0 (paging), nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1542] Also activating Harakiri-alarm (SIGALRM=14) to expire after 500s elapsed to prevent hangs, nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1544] Harakiri signal handler 'signal_harakiri' for signal#14 (SIGALRM) installed at 0x81f3c0 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1617] Signal#15 was caused by unrecognized si_code [memaddr=0x1], nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1686] Starting DrHook backtrace for signal#15, nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3843] 4032 MB (maxheap), 4403 MB (maxrss), 0 MB (maxstack)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : MASTER
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT0
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT2
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT3
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.094] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT4
double free or corruption (top)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.095] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : STEPO
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.095] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : SCAN2M
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.095] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : GP_MODEL_HEAP
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.095] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : GP_MODEL
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.095] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : RADDRV
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.095] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : RADINTG
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.095] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : >OMP-RADINTG-RADLSW (1210)
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.095] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : RADIATION_SCHEME
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.095] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : radiation_interface:radiation
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.095] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : radiation_mcica_sw:solver_mcica_sw
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212515:1674591915:196.095] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : radiation_cloud_generator:cloud_generator
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212516:1674591916:197.095] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1734] DrHook backtrace done for signal#15, nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2953:2953] [20230124:212516:1674591916:197.095] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1785] Calling previous signal handler at 0x1ce8bb0 for signal#15, nsigs = 1
forrtl: error (78): process killed (SIGTERM)
Image PC Routine Line Source
oifs_43r3_model.e 0000000001CE916B Unknown Unknown Unknown
oifs_43r3_model.e 000000000081FFF1 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DC9090 Unknown Unknown Unknown
oifs_43r3_model.e 000000000185E45A radiation_pdf_sam 143 radiation_pdf_sampler.F90
oifs_43r3_model.e 000000000196FB80 radiation_cloud_g 198 radiation_cloud_generator.F90
oifs_43r3_model.e 0000000001969EC8 radiation_mcica_s 212 radiation_mcica_sw.F90
oifs_43r3_model.e 0000000001862DC8 radiation_interfa 419 radiation_interface.F90
oifs_43r3_model.e 00000000012D30E5 radiation_scheme_ 484 radiation_scheme.F90
oifs_43r3_model.e 000000000129A415 radintg_ 1455 radintg.F90
oifs_43r3_model.e 0000000001270124 raddrv_ 537 raddrv.F90
oifs_43r3_model.e 0000000000BFD839 gp_model_ 482 gp_model.F90
oifs_43r3_model.e 00000000012E9482 gp_model_heap_ 74 gp_model_heap.F90
oifs_43r3_model.e 0000000000BBC1CE scan2m_ 535 scan2m.F90
oifs_43r3_model.e 000000000057A664 stepo_ 327 stepo.F90
oifs_43r3_model.e 000000000055DEEC cnt4_ 1133 cnt4.F90
oifs_43r3_model.e 00000000005471C9 cnt3_ 267 cnt3.F90
oifs_43r3_model.e 0000000000546412 cnt2_ 88 cnt2.F90
oifs_43r3_model.e 0000000000545E28 cnt1_ 92 cnt1.F90
oifs_43r3_model.e 000000000040708D cnt0_ 146 cnt0.F90
oifs_43r3_model.e 000000000040220F MAIN__ 96 master.F90
oifs_43r3_model.e 00000000004021A2 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DCA390 Unknown Unknown Unknown
oifs_43r3_model.e 000000000040206E Unknown Unknown Unknown
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2018050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0820
(argv4) batchid: 987
(argv5) wuid: 12204464
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/4
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/4/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/4/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_467d28a54f765caa463421df45eb7ad9 to: /var/lib/boinc-client/slots/4/oifs_43r3_ps_0820_2018050100_123_987_12204464.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/4/oifs_43r3_ps_0820_2018050100_123_987_12204464.zip
ic_ancil_file: ic_ancil_12204464
ifsdata_file: ifsdata_12204464
climate_data_file: clim_data_12204464
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_8ead1d3ea42fb9993f857c72ad5b7eac to: /var/lib/boinc-client/slots/4/ic_ancil_12204464.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/4/ic_ancil_12204464.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_61800d0db66ca28702ed0afc14a0c793 to: /var/lib/boinc-client/slots/4/ifsdata/ifsdata_12204464.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/4/ifsdata/ifsdata_12204464.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/4/159l_2/clim_data_12204464.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/4/159l_2/clim_data_12204464.zip
Checking for progress XML file: /var/lib/boinc-client/slots/4/progress_file_12204464.xml
Opened progress file ok : /var/lib/boinc-client/slots/4/progress_file_12204464.xml
-- Model is restarting --
Adjusting last_iter, 1977, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/4/progress_file_12204464.xml
last_cpu_time: 53377
upload_file_number: 82
last_iter: 1967
last_upload: 7084800
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0820_2018050100_123_987_12204464_0_r973255285
The child process has been launched with process id: 2930
Executing the command: /var/lib/boinc-client/slots/4/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212819:1674592099:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
21:28:56 STEP 1968 H=1968:00 +CPU= 28.736
21:29:14 STEP 1969 H=1969:00 +CPU= 17.122
21:29:30 STEP 1970 H=1970:00 +CPU= 16.253
21:29:54 STEP 1971 H=1971:00 +CPU= 23.232
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1538] Received signal#15 (SIGTERM) :: 2467MB (heap), 4403MB (maxrss), 0MB (maxstack), 0 (paging), nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1542] Also activating Harakiri-alarm (SIGALRM=14) to expire after 500s elapsed to prevent hangs, nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1544] Harakiri signal handler 'signal_harakiri' for signal#14 (SIGALRM) installed at 0x81f3c0 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1617] Signal#15 was caused by unrecognized si_code [memaddr=0x1], nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1686] Starting DrHook backtrace for signal#15, nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3843] 2473 MB (maxheap), 4403 MB (maxrss), 0 MB (maxstack)
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : MASTER
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT0
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT2
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT3
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT4
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : GPNORM_GFL
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212954:1674592194:94.622] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : >OMP-GPNORM1 (1427)
double free or corruption (top)
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212955:1674592195:95.623] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1734] DrHook backtrace done for signal#15, nsigs = 1
[EC_DRHOOK:philip-All-Series:1:1:2930:2930] [20230124:212955:1674592195:95.623] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1785] Calling previous signal handler at 0x1ce8bb0 for signal#15, nsigs = 1
forrtl: error (78): process killed (SIGTERM)
Image PC Routine Line Source
oifs_43r3_model.e 0000000001CE916B Unknown Unknown Unknown
oifs_43r3_model.e 000000000081FFF1 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DC9090 Unknown Unknown Unknown
oifs_43r3_model.e 0000000000A6FEFC gpnorm_gfl_ 159 gpnorm_gfl.F90
oifs_43r3_model.e 000000000055D47D cnt4_ 833 cnt4.F90
oifs_43r3_model.e 00000000005471C9 cnt3_ 267 cnt3.F90
oifs_43r3_model.e 0000000000546412 cnt2_ 88 cnt2.F90
oifs_43r3_model.e 0000000000545E28 cnt1_ 92 cnt1.F90
oifs_43r3_model.e 000000000040708D cnt0_ 146 cnt0.F90
oifs_43r3_model.e 000000000040220F MAIN__ 96 master.F90
oifs_43r3_model.e 00000000004021A2 Unknown Unknown Unknown
oifs_43r3_model.e 0000000001DCA390 Unknown Unknown Unknown
oifs_43r3_model.e 000000000040206E Unknown Unknown Unknown
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.05_x86_64-pc-linux-gnu
(argv1) start_date: 2018050100
(argv2) exptid: hq0f
(argv3) unique_member_id: 0820
(argv4) batchid: 987
(argv5) wuid: 12204464
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/4
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.05
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/4/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/4/oifs_43r3_ps_app_1.05_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_467d28a54f765caa463421df45eb7ad9 to: /var/lib/boinc-client/slots/4/oifs_43r3_ps_0820_2018050100_123_987_12204464.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/4/oifs_43r3_ps_0820_2018050100_123_987_12204464.zip
ic_ancil_file: ic_ancil_12204464
ifsdata_file: ifsdata_12204464
climate_data_file: clim_data_12204464
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
nfrres: restart dump frequency (steps) 24
Copying IC ancils from: ../../projects/climateprediction.net/jf_8ead1d3ea42fb9993f857c72ad5b7eac to: /var/lib/boinc-client/slots/4/ic_ancil_12204464.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/4/ic_ancil_12204464.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_61800d0db66ca28702ed0afc14a0c793 to: /var/lib/boinc-client/slots/4/ifsdata/ifsdata_12204464.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/4/ifsdata/ifsdata_12204464.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/4/159l_2/clim_data_12204464.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/4/159l_2/clim_data_12204464.zip
Checking for progress XML file: /var/lib/boinc-client/slots/4/progress_file_12204464.xml
Opened progress file ok : /var/lib/boinc-client/slots/4/progress_file_12204464.xml
-- Model is restarting --
Adjusting last_iter, 1971, to previous model restart step.
Creating progress file: /var/lib/boinc-client/slots/4/progress_file_12204464.xml
last_cpu_time: 53459
upload_file_number: 82
last_iter: 1967
last_upload: 7084800
model_completed: 0
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0820_2018050100_123_987_12204464_0_r973255285
The child process has been launched with process id: 2896
Executing the command: /var/lib/boinc-client/slots/4/oifs_43r3_model.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:philip-All-Series:1:1:2896:2896] [20230124:213303:1674592383:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
21:33:49 STEP 1968 H=1968:00 +CPU= 29.030
21:34:05 STEP 1969 H=1969:00 +CPU= 15.098
21:34:20 STEP 1970 H=1970:00 +CPU= 15.123
21:34:44 STEP 1971 H=1971:00 +CPU= 23.458
21:35:01 STEP 1972 H=1972:00 +CPU= 16.795
21:35:18 STEP 1973 H=1973:00 +CPU= 16.829
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
21:37:59 STEP 1974 H=1974:00 +CPU= 32.286
21:38:19 STEP 1975 H=1975:00 +CPU= 19.375
21:38:39 STEP 1976 H=1976:00 +CPU= 19.297
21:39:13 STEP 1977 H=1977:00 +CPU= 31.644
21:39:35 STEP 1978 H=1978:00 +CPU= 20.442
21:39:54 STEP 1979 H=1979:00 +CPU= 18.643
21:40:26 STEP 1980 H=1980:00 +CPU= 31.357
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMGGhq0f+001980
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMSHhq0f+001980
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMUAhq0f+001980
21:40:44 STEP 1981 H=1981:00 +CPU= 18.092
21:41:03 STEP 1982 H=1982:00 +CPU= 18.553
21:41:32 STEP 1983 H=1983:00 +CPU= 28.502
21:41:51 STEP 1984 H=1984:00 +CPU= 18.824
21:42:10 STEP 1985 H=1985:00 +CPU= 18.492
Suspend request received from the BOINC client, suspending the child process
Resuming the child process
21:42:44 STEP 1986 H=1986:00 +CPU= 27.813
21:43:02 STEP 1987 H=1987:00 +CPU= 17.830
21:43:19 STEP 1988 H=1988:00 +CPU= 16.454
21:43:44 STEP 1989 H=1989:00 +CPU= 24.439
21:44:00 STEP 1990 H=1990:00 +CPU= 15.667
21:44:15 STEP 1991 H=1991:00 +CPU= 14.852
21:44:41 STEP 1992 H=1992:00 +CPU= 26.231
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMGGhq0f+001992
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMSHhq0f+001992
Moving to projects directory: /var/lib/boinc-client/slots/4/ICMUAhq0f+001992
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001968
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001968
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001968
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMGGhq0f+001980
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMSHhq0f+001980
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12204464/ICMUAhq0f+001980
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0820_2018050100_123_987_12204464_0_r973255285_82.zip
Uploading the intermediate file: upload_file_82.zip
21:44:56 STEP 1993 H=1993:00 +CPU= 14.502
21:45:11 STEP 1994 H=1994:00 +CPU= 15.025
21:45:34 STEP 1995 H=1995:00 +CPU= 22.629
21:45:49 STEP 1996 H=1996:00 +CPU= 15.218
21:46:05 STEP 1997 H=1997:00 +CPU= 15.411
21:46:30 STEP 1998 H=1998:00 +CPU= 24.085
21:46:46 STEP 1999 H=1999:00 +CPU= 15.848
21:47:02 STEP 2000 H=2000:00 +CPU= 15.877
21:47:26 STEP 2001 H=2001:00 +CPU= 24.070
21:47:43 STEP 2002 H=2002:00 +CPU= 15.844
21:47:59 STEP 2003 H=2003:00 +CPU= 15.510
Abort request received from BOINC client, ending the child process
..Failed, model did not complete successfully
</stderr_txt>
]]>
