Why the directory size doesn't shrink after rm all files in the directory ?

Issue

1. Create a test directory se_test and check its size:

[root@*** test]# mkdir se_test
[root@*** test]# ls -al
total 16
drwxr-xr-x  3 root root 4096 Jul 27 16:57 .
drwxrwxrwt 41 root root 4096 Jul 27 16:57 ..
drwxr-xr-x  2 root root 4096 Jul 27 16:57 se_test
[root@*** test]# cd se_test
[root@*** se_test]# ls -al                                    
total 8
drwxr-xr-x 2 root root 4096 Jul 27 16:57 .
drwxr-xr-x 3 root root 4096 Jul 27 16:57 ..

Now the size of se_test is 4096

2.Create 1000 files and check the size of directory:

[root@*** se_test]# for i in {1..1000}; do touch $i.txt ; done
[root@*** se_test]# ls -al
total 32
drwxr-xr-x 2 root root 28672 Jul 27 17:06 .
drwxr-xr-x 3 root root  4096 Jul 27 16:57 ..
-rw-r--r-- 1 root root     0 Jul 27 17:06 1000.txt
-rw-r--r-- 1 root root     0 Jul 27 17:06 100.txt
...etc...

[root@*** se_test]# cd ..
[root@*** test]# ls -al
total 40
drwxr-xr-x  3 root root  4096 Jul 27 16:57 .
drwxrwxrwt 41 root root  4096 Jul 27 16:57 ..
drwxr-xr-x  2 root root 28672 Jul 27 17:06 se_test
[root@*** test]# cd se_test 
[root@*** se_test]# ls |wc -l
1000

Now the directory size is 28672Bit

3.Remove all the file in the test directory se_test:

[root@*** se_test]# rm -rf *
[root@*** se_test]# ls -al
total 32
drwxr-xr-x 2 root root 28672 Jul 27 17:07 .
drwxr-xr-x 3 root root  4096 Jul 27 16:57 ..
[root@*** se_test]# cd ..
[root@*** test]# ls -al
total 40
drwxr-xr-x  3 root root  4096 Jul 27 16:57 .
drwxrwxrwt 41 root root  4096 Jul 27 16:57 ..
drwxr-xr-x  2 root root 28672 Jul 27 17:07 se_test

But the directory size doesn't change at all.