bash script- 자바 트러블 슈팅 (Java Trouble Shooting) 용

 

리눅스에서 동작하는 자바 어플리케이션 (톰캣) 을 실행하다 문제가 생겼을 때 써먹는 스크립트.

기본적인 시스템 정보와 자바 heap,native heap, 자바 쓰레드 정보, 자바 쓰레드 중에 cpu 많이 먹는 녀석등을 출력한다.

#!/bin/bash if [ -z "$JAVA_HOME" ] then     echo "\$JAVA_HOME must be defined"     exit -1 fi if [ $# -eq 0  ]; then     echo "Invalid argument count. Please input $pid of java process"     exit -1 fi pid=$1 jstackBin="$JAVA_HOME/bin/jstack" jmapBin="$JAVA_HOME/bin/jmap" dateStr=`date '+%Y%m%d_%H%M%S'` dumpFile=threadDump.$dateStr # init touch dumpFile echo $dateStr > $dumpFile echo "" >> $dumpFile echo "### ps" >> $dumpFile echo "" >> $dumpFile ps -ef | grep $pid | grep -v grep >> $dumpFile # top result echo "" >> $dumpFile echo "### top result" >> $dumpFile echo "" >> $dumpFile top -cb -n 1 | head -n 12 >> $dumpFile # all cpu ratio echo "" >> $dumpFile echo "### all cpu (sar -P ALL)" >> $dumpFile echo "" >> $dumpFile sar -P ALL 1 1 >> $dumpFile # runq echo "" >> $dumpFile echo "### runq (sar -q)" >> $dumpFile echo "" >> $dumpFile sar -q 1 1 >> $dumpFile # all netstat echo "" >> $dumpFile echo "### netsat " >> $dumpFile echo "" >> $dumpFile netstat -an >> $dumpFile # cpu ratio of java thread echo "" >> $dumpFile echo "### cpu ratio of java thread" >> $dumpFile echo "" >> $dumpFile echo "cpu  thread-id" >> $dumpFile ps -mo pcpu,lwp -p31527 | awk '{if ($2 ~/[0-9]/) {print $2  "% " $1""}}' | sort -k 1 -r >> $dumpFile 2>&1 echo "" >> $dumpFile # heap memory (native heap + java heap) echo "" >> $dumpFile echo "### native heap" >> $dumpFile echo "" >> $dumpFile ps -p $pid -o pid,vsz,rss,drs,%mem >> $dumpFile 2>&1 echo "" >> $dumpFile # thread dump echo "" >> $dumpFile echo "### thread dump" >> $dumpFile echo "" >> $dumpFile $jstackBin -F $pid >> $dumpFile 2>&1 # jmap  heap summary echo "" >> $dumpFile echo "### heap summary " >> $dumpFile echo "" >> $dumpFile $jmapBin -F -heap $pid >> $dumpFile 2>&1 # jmap histogram echo "" >> $dumpFile echo "### heap historam " >> $dumpFile echo "" >> $dumpFile $jmapBin -F -histo $pid | head -n 200 >> $dumpFile 2>&1 echo "system & java data dump : $dumpFile"

 

이 코드를 이용하면, 다음과 같은 java 어플리케이션은 다음과 같이 출력된다.

 

* 자바 어플리케이션

class RunnableThread implements Runnable {         Thread runner;         public RunnableThread() {         }         public RunnableThread(String threadName) {                 runner = new Thread(this, threadName); // (1) Create a new thread.                 System.out.println(runner.getName());                 runner.start(); // (2) Start the thread.         }         public void run() {                 //Display info about this particular thread                 System.out.println(Thread.currentThread());         } } public class RunnableExample {         public static void main(String[] args) {                 Thread thread1 = new Thread(new RunnableThread(), "thread1");                 Thread thread2 = new Thread(new RunnableThread(), "thread2");                 RunnableThread thread3 = new RunnableThread("thread3");                 //Start the threads                 thread1.start();                 thread2.start();                 try {                         while(true) {                         //delay for one second                         Thread.currentThread().sleep(0); ;                         }                 } catch (InterruptedException e) {                 }                 //Display info about the main thread                 System.out.println(Thread.currentThread());         } }

 

* dump 파일

./dump.sh java프로세스id

### ps www       8055  6211  0 19:21 pts/1    00:00:00 /bin/bash ./test.sh 31527 www      31527 30883 99 Feb01 pts/0    1-01:05:58 /home/www/jdk6/bin/java RunnableExample ### top result top - 19:21:20 up 36 days,  4:44,  2 users,  load average: 1.00, 1.00, 1.00 Tasks: 162 total,   1 running, 161 sleeping,   0 stopped,   0 zombie Cpu(s):  0.1%us,  0.2%sy,  0.0%ni, 99.7%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st Mem:  12273860k total,  4505204k used,  7768656k free,   375084k buffers Swap:  2096472k total,        0k used,  2096472k free,  3825760k cached   PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND                                           31527 www       15   0 1171m  12m 6504 S 99.4  0.1   1505:59 /home/www/jdk6/bin/java RunnableExample               1 root      15   0 10344  680  572 S  0.0  0.0   0:01.37 init [3]                                              2 root      RT  -5     0    0    0 S  0.0  0.0   0:00.04 [migration/0]                                         3 root      34  19     0    0    0 S  0.0  0.0   0:00.02 [ksoftirqd/0]                                         4 root      RT  -5     0    0    0 S  0.0  0.0   0:00.00 [watchdog/0]                                      ### all cpu (sar -P ALL) Linux 2.6.18-164.el5 (blgdbs101)        2012³â 02¿ù 02ÀÏ 19½Ã 21ºÐ 20ÃÊ       CPU     %user     %nice   %system   %iowait    %steal     %idle 19½Ã 21ºÐ 21ÃÊ       all      2.50      0.00      5.83      0.00      0.00     91.67 19½Ã 21ºÐ 21ÃÊ         0     30.00      0.00     70.00      0.00      0.00      0.00 19½Ã 21ºÐ 21ÃÊ         1      0.00      0.00      0.00      0.00      0.00    100.00 19½Ã 21ºÐ 21ÃÊ         2      0.00      0.00      0.00      0.00      0.00    100.00 19½Ã 21ºÐ 21ÃÊ         3      0.00      0.00      0.00      0.00      0.00    100.00 19½Ã 21ºÐ 21ÃÊ         4      0.00      0.00      0.00      0.00      0.00    100.00 19½Ã 21ºÐ 21ÃÊ         5      0.00      0.00      0.00      0.00      0.00    100.00 19½Ã 21ºÐ 21ÃÊ         6      0.00      0.00      0.00      0.00      0.00    100.00 19½Ã 21ºÐ 21ÃÊ         7      0.00      0.00      0.00      0.00      0.00    100.00 19½Ã 21ºÐ 21ÃÊ         8      0.00      0.00      0.00      0.00      0.00    100.00 19½Ã 21ºÐ 21ÃÊ         9      0.00      0.00      0.00      0.00      0.00    100.00 19½Ã 21ºÐ 21ÃÊ        10      0.00      0.00      0.99      0.00      0.00     99.01 19½Ã 21ºÐ 21ÃÊ        11      0.00      0.00      0.00      0.00      0.00    100.00 Average:          CPU     %user     %nice   %system   %iowait    %steal     %idle Average:          all      2.50      0.00      5.83      0.00      0.00     91.67 Average:            0     30.00      0.00     70.00      0.00      0.00      0.00 Average:            1      0.00      0.00      0.00      0.00      0.00    100.00 Average:            2      0.00      0.00      0.00      0.00      0.00    100.00 Average:            3      0.00      0.00      0.00      0.00      0.00    100.00 Average:            4      0.00      0.00      0.00      0.00      0.00    100.00 Average:            5      0.00      0.00      0.00      0.00      0.00    100.00 Average:            6      0.00      0.00      0.00      0.00      0.00    100.00 Average:            7      0.00      0.00      0.00      0.00      0.00    100.00 Average:            8      0.00      0.00      0.00      0.00      0.00    100.00 Average:            9      0.00      0.00      0.00      0.00      0.00    100.00 Average:           10      0.00      0.00      0.99      0.00      0.00     99.01 Average:           11      0.00      0.00      0.00      0.00      0.00    100.00 ### runq (sar -q) Linux 2.6.18-164.el5 (blgdbs101)        2012³â 02¿ù 02ÀÏ 19½Ã 21ºÐ 21ÃÊ   runq-sz  plist-sz   ldavg-1   ldavg-5  ldavg-15 19½Ã 21ºÐ 22ÃÊ         1       233      1.00      1.00      1.00 Average:            1       233      1.00      1.00      1.00 ### netsat Active Internet connections (servers and established) Proto Recv-Q Send-Q Local Address               Foreign Address             State      tcp        0      0 0.0.0.0:10560               0.0.0.0:*                   LISTEN      tcp        0      0 0.0.0.0:544                 0.0.0.0:*                   LISTEN      tcp        0      0 0.0.0.0:10561               0.0.0.0:*                   LISTEN      tcp        0      0 0.0.0.0:6051                0.0.0.0:*                   LISTEN      tcp        0      0 0.0.0.0:873                 0.0.0.0:*                   LISTEN      tcp        0      0 0.0.0.0:22                  0.0.0.0:*                   LISTEN      tcp        0      0 0.0.0.0:2105                0.0.0.0:*                   LISTEN      tcp        0      0 0.0.0.0:543                 0.0.0.0:*                   LISTEN      tcp        0      0 1.2.80.92:2105            1.2.22.70:53828           ESTABLISHED tcp        0      0 1.2.80.92:55785           1.2.235.21:35001          ESTABLISHED tcp        0      0 1.2.80.92:2105            1.2.22.70:39864           ESTABLISHED tcp        0      0 1.2.80.92:53006           1.2.235.22:35001          ESTABLISHED udp    51912      0 0.0.0.0:333                 0.0.0.0:*                               udp        0      0 0.0.0.0:421                0.0.0.0:*                               udp        0      0 0.0.0.0:10241               0.0.0.0:*                               udp        0      0 1.2.80.92:142             0.0.0.0:*                               udp        0      0 127.0.0.1:22               0.0.0.0:*                               udp        0      0 0.0.0.0:121                 0.0.0.0:*                               Active UNIX domain sockets (servers and established) Proto RefCnt Flags       Type       State         I-Node Path unix  2      [ ACC ]     STREAM     LISTENING     7636   /opt/nbp/noms/var/master unix  2      [ ]         DGRAM                    2964   @/org/kernel/udev/udevd unix  14     [ ]         DGRAM                    7046   /dev/log unix  2      [ ]         DGRAM                    4980259 unix  2      [ ]         DGRAM                    4980239 unix  2      [ ]         DGRAM                    4843655 unix  2      [ ]         DGRAM                    4843635 unix  2      [ ]         DGRAM                    4835066 unix  2      [ ]         DGRAM                    1236651 unix  3      [ ]         STREAM     CONNECTED     7661   /opt/nbp/noms/var/master unix  3      [ ]         STREAM     CONNECTED     7660   unix  2      [ ]         DGRAM                    7634   unix  2      [ ]         DGRAM                    7543   unix  2      [ ]         DGRAM                    7344   unix  2      [ ]         DGRAM                    7309   unix  2      [ ]         DGRAM                    7286   unix  2      [ ]         DGRAM                    7055   ### cpu ratio of java thread cpu  thread-id 31546% 0.0 31545% 0.0 31544% 0.0 31543% 0.0 31542% 0.0 31541% 0.0 31540% 0.0 31539% 0.0 31538% 0.0 31537% 0.0 31536% 0.0 31535% 0.0 31534% 0.0 31533% 0.0 31532% 0.0 31531% 0.0 31530% 0.0 31529% 0.0 31528% 99.9 31527% 0.0 ### native heap   PID    VSZ   RSS   DRS %MEM 31527 1199636 12372 1199598  0.1 ### thread dump Attaching to process ID 31527, please wait... Debugger attached successfully. Server compiler detected. JVM version is 19.1-b02 Deadlock Detection: No deadlocks found. Thread 31542: (state = BLOCKED) Thread 31541: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Interpreted frame) - java.lang.ref.ReferenceQueue.remove(long) @bci=44, line=118 (Interpreted frame) - java.lang.ref.ReferenceQueue.remove() @bci=2, line=134 (Interpreted frame) - java.lang.ref.Finalizer$FinalizerThread.run() @bci=3, line=159 (Interpreted frame) Thread 31540: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Interpreted frame) - java.lang.Object.wait() @bci=2, line=485 (Interpreted frame) - java.lang.ref.Reference$ReferenceHandler.run() @bci=46, line=116 (Interpreted frame) Thread 31528: (state = IN_VM) - java.lang.Thread.sleep(long) @bci=0 (Compiled frame; information may be imprecise) - RunnableExample.main(java.lang.String[]) @bci=57, line=30 (Compiled frame)   ### heap summary Attaching to process ID 31527, please wait... Debugger attached successfully. Server compiler detected. JVM version is 19.1-b02 using thread-local object allocation. Parallel GC with 10 thread(s) Heap Configuration:    MinHeapFreeRatio = 40    MaxHeapFreeRatio = 70    MaxHeapSize      = 1073741824 (1024.0MB)    NewSize          = 1048576 (1.0MB)    MaxNewSize       = 4294901760 (4095.9375MB)    OldSize          = 4194304 (4.0MB)    NewRatio         = 2    SurvivorRatio    = 8    PermSize         = 16777216 (16.0MB)    MaxPermSize      = 67108864 (64.0MB) Heap Usage: PS Young Generation Eden Space:    capacity = 16842752 (16.0625MB)    used     = 1347584 (1.28515625MB)    free     = 15495168 (14.77734375MB)    8.000972762645915% used From Space:    capacity = 2752512 (2.625MB)    used     = 0 (0.0MB)    free     = 2752512 (2.625MB)    0.0% used To Space:    capacity = 2752512 (2.625MB)    used     = 0 (0.0MB)    free     = 2752512 (2.625MB)    0.0% used PS Old Generation    capacity = 44761088 (42.6875MB)    used     = 0 (0.0MB)    free     = 44761088 (42.6875MB)    0.0% used PS Perm Generation    capacity = 16777216 (16.0MB)    used     = 2069280 (1.973419189453125MB)    free     = 14707936 (14.026580810546875MB)    12.333869934082031% used ### heap historam Object Histogram: num       #instances    #bytes  Class description -------------------------------------------------------------------------- 1:              4831    483496  * ConstMethodKlass 2:              8140    396296  * SymbolKlass 3:              4831    388832  * MethodKlass 4:              1886    224232  char[] 5:              320     187152  * ConstantPoolKlass 6:              320     130496  * InstanceKlassKlass 7:              294     127056  * ConstantPoolCacheKlass 8:              516     91664   byte[] 9:              1743    41832   java.lang.String 10:             390     37440   java.lang.Class 11:             529     31304   * System ObjArray 12:             480     31168   short[] 13:             743     23776   java.util.TreeMap$Entry 14:             365     19472   int[] 15:             327     14000   java.lang.Object[] 16:             41      13448   * ObjArrayKlassKlass 17:             76      6080    java.lang.reflect.Method 18:             183     5760    java.lang.String[] 19:             96      4608    java.nio.HeapCharBuffer 20:             95      4560    java.nio.HeapByteBuffer 21:             40      2880    java.lang.reflect.Field 22:             8       2624    * TypeArrayKlassKlass 23:             113     2256    java.lang.Class[] 24:             8       1856    * MethodDataKlass 25:             58      1392    java.util.Hashtable$Entry 26:             17      1360    java.util.HashMap$Entry[] 27:             12      1344    * KlassKlass 28:             71      1136    java.lang.StringBuilder 29:             10      1040    java.util.Hashtable$Entry[] 30:             8       896     java.lang.Thread 31:             13      728     java.net.URL 32:             19      608     java.util.Locale 33:             15      600     java.util.HashMap 34:             35      560     java.io.File 35:             16      512     java.util.concurrent.ConcurrentHashMap$Segment 36:             8       512     java.lang.reflect.Constructor 37:             19      496     java.util.concurrent.ConcurrentHashMap$HashEntry[] 38:             20      480     java.util.concurrent.ConcurrentHashMap$HashEntry 39:             13      416     java.lang.ref.SoftReference 40:             13      416     java.util.LinkedHashMap$Entry 41:             16      384     java.util.concurrent.locks.ReentrantLock$NonfairSync 42:             14      336     java.util.HashMap$Entry 43:             10      320     java.io.ObjectStreamField 44:             2       320     java.lang.reflect.Method[] 45:             12      288     java.security.AccessControlContext 46:             5       280     sun.nio.cs.ext.EUC_KR$Decoder 47:             11      264     java.io.ExpiringCache$Entry 48:             16      256     sun.security.action.GetPropertyAction 49:             6       240     java.util.Hashtable 50:             7       224     java.lang.ref.Finalizer 51:             9       216     java.net.Parts 52:             8       192     java.lang.OutOfMemoryError 53:             4       192     java.util.TreeMap 54:             4       184     java.lang.reflect.Field[] 55:             3       168     sun.nio.cs.ext.EUC_KR$Encoder 56:             7       168     java.util.Vector 57:             4       160     sun.misc.URLClassPath$JarLoader 58:             8       152     java.io.ObjectStreamField[] 59:             9       144     java.lang.StringBuffer 60:             6       144     java.util.ArrayList 61:             15      120     java.lang.Object 62:             1       112     java.lang.ref.Reference$ReferenceHandler 63:             2       112     java.io.ExpiringCache$1 64:             1       112     java.lang.ref.Finalizer$FinalizerThread 65:             6       96      java.io.FileDescriptor 66:             2       96      java.lang.ThreadGroup 67:             4       96      sun.reflect.NativeConstructorAccessorImpl 68:             2       96      sun.nio.cs.StreamEncoder 69:             1       96      sun.net.www.protocol.file.FileURLConnection 70:             1       80      java.util.concurrent.ConcurrentHashMap$Segment[] 71:             1       80      java.lang.ThreadLocal$ThreadLocalMap$Entry[] 72:             5       80      sun.misc.URLClassPath$3 73:             2       80      java.io.BufferedWriter 74:             2       80      java.util.StringTokenizer 75:             1       72      sun.misc.Launcher$ExtClassLoader 76:             3       72      java.lang.ref.WeakReference 77:             1       72      sun.misc.Launcher$AppClassLoader 78:             3       72      sun.nio.cs.Surrogate$Parser 79:             3       72      java.util.Stack 80:             3       72      java.lang.RuntimePermission 81:             3       72      java.lang.NoSuchMethodError 82:             4       64      java.net.URLClassLoader$1 83:             2       64      java.io.FilePermission 84:             4       64      java.lang.Class$1 85:             4       64      java.lang.reflect.Constructor[] 86:             2       64      java.security.PrivilegedActionException 87:             4       64      sun.reflect.DelegatingConstructorAccessorImpl 88:             2       64      sun.misc.URLClassPath$FileLoader$1 89:             2       64      java.io.ExpiringCache 90:             2       64      java.lang.ClassNotFoundException 91:             2       64      java.lang.Thread[] 92:             2       64      java.lang.ThreadLocal$ThreadLocalMap$Entry 93:             4       64      sun.misc.MetaIndex 94:             2       64      java.io.PrintStream 95:             4       64      java.io.FileInputStream 96:             2       64      sun.misc.URLClassPath 97:             2       64      java.security.CodeSource 98:             3       56      java.io.File[] 99:             2       48      java.net.URL[] 100:            2       48      java.lang.ref.ReferenceQueue 101:            1       48      java.util.Properties 102:            2       48      java.security.Permissions 103:            2       48      java.io.BufferedOutputStream 104:            2       48      java.lang.ref.ReferenceQueue$Null 105:            1       48      java.io.BufferedReader 106:            3       48      sun.misc.Signal 107:            3       48      java.security.ProtectionDomain[] 108:            1       48      long[] 109:            3       48      java.lang.ThreadLocal 110:            2       48      java.io.OutputStreamWriter 111:            3       48      java.nio.charset.CodingErrorAction 112:            2       48      java.io.FileOutputStream 113:            3       48      RunnableThread 114:            3       48      java.lang.Boolean 115:            3       48      java.lang.Integer 116:            1       40      sun.nio.cs.StandardCharsets$Cache 117:            1       40      sun.nio.cs.StandardCharsets$Aliases 118:            1       40      sun.nio.cs.StreamDecoder 119:            1       40      java.util.concurrent.ConcurrentHashMap 120:            1       40      sun.nio.cs.StandardCharsets$Classes 121:            5       40      java.lang.Class$3 122:            2       32      java.util.HashSet 123:            1       32      java.lang.ClassLoader$NativeLibrary 124:            1       32      java.lang.OutOfMemoryError[] 125:            1       32      sun.misc.SoftCache 126:            4       32      java.lang.ref.ReferenceQueue$Lock 127:            2       32      java.io.FilePermission$1 128:            1       32      java.util.concurrent.atomic.AtomicReferenceFieldUpdater$AtomicReferenceFieldUpdaterImpl 129:            2       32      sun.misc.NativeSignalHandler 130:            2       32      java.nio.ByteOrder 131:            1       32      java.lang.ThreadGroup[] 132:            1       32      java.util.Collections$SynchronizedMap 133:            1       32      sun.nio.cs.ext.ExtendedCharsets 134:            4       32      sun.reflect.ReflectionFactory$GetReflectionFactoryAction 135:            1       32      java.security.ProtectionDomain 136:            1       32      java.io.BufferedInputStream 137:            2       32      java.nio.charset.CoderResult 138:            1       24      java.io.FileReader 139:            1       24      sun.net.www.MessageHeader 140:            1       24      java.security.BasicPermissionCollection 141:            3       24      sun.reflect.ReflectionFactory$1 142:            1       24      java.io.UnixFileSystem 143:            1       24      java.lang.StringCoding$StringDecoder 144:            1       24      java.lang.NullPointerException 145:            1       24      java.lang.VirtualMachineError 146:            1       24      java.lang.reflect.ReflectPermission 147:            1       24      sun.misc.Launcher$AppClassLoader$1 148:            1       24      sun.nio.cs.StandardCharsets 149:            1       24      java.lang.ThreadLocal$ThreadLocalMap 150:            1       24      java.util.BitSet 151:            1       24      java.lang.ArithmeticException 152:            1       24      java.lang.StringCoding$StringEncoder 153:            1       24      sun.nio.cs.ext.EUC_KR 154:            1       24      java.lang.ref.Reference 155:            1       16      java.nio.charset.CoderResult$1 156:            1       16      java.util.Collections$EmptyMap 157:            1       16      sun.misc.Launcher$ExtClassLoader$1 158:            2       16      sun.net.www.protocol.jar.Handler 159:            1       16      java.io.FilePermissionCollection 160:            1       16      java.security.Principal[] 161:            1       16      java.util.Collections$EmptyList 162:            1       16      java.lang.StackTraceElement[] 163:            1       16      java.security.cert.Certificate[] 164:            1       16      java.security.Policy$UnsupportedEmptyCollection 165:            1       16      sun.jkernel.DownloadManager$1 166:            1       16      java.util.concurrent.atomic.AtomicInteger 167:            1       16      sun.misc.URLClassPath$FileLoader 168:            1       16      java.nio.charset.CoderResult$2 169:            1       16      sun.misc.Launcher 170:            1       16      java.security.ProtectionDomain$Key 171:            1       16      java.lang.SystemClassLoaderAction 172:            1       16      java.lang.ClassLoader$3 173:            1       8       java.lang.reflect.ReflectAccess 174:            1       8       java.net.UnknownContentHandler 175:            1       8       sun.net.www.protocol.file.Handler 176:            1       8       java.lang.System$2 177:            1       8       java.security.ProtectionDomain$2 178:            1       8       sun.misc.Unsafe 179:            1       8       java.util.Hashtable$EmptyIterator 180:            1       8       java.util.Collections$ReverseComparator 181:            1       8       java.net.URLClassLoader$7 182:            1       8       java.security.AccessControlContext$1 183:            1       8       java.lang.ref.Reference$Lock 184:            1       8       java.io.FileDescriptor$1 185:            1       8       sun.misc.Launcher$Factory 186:            1       8       java.lang.Terminator$1 187:            1       8       java.lang.Compiler$1 188:            1       8       java.util.Hashtable$EmptyEnumerator 189:            1       8       sun.jkernel.DownloadManager$2 190:            1       8       java.lang.Runtime 191:            1       8       java.nio.charset.Charset$3 192:            1       8       java.lang.String$CaseInsensitiveComparator 193:            1       8       java.util.Collections$EmptySet 194:            1       8       sun.reflect.ReflectionFactory 195:            1       8       sun.misc.ASCIICaseInsensitiveComparator Total :         27128   2294784