OSS 文件分析

文件大小分布
目录最后修改时间（确定是否长期未使用）
目录总容量

脚本：

#!/bin/bash
# 分析目标下的所有子目录

target=$1

time_start=$(date +%s)
entry=$(ossutil64 ls -d $target | grep "^oss://")
printf "%-32s%16s%16s%16s%16s%16s%10s%9s%20s\n" "DIR" "<100KB" "<300KB" "<1MB" "<5MB" ">5MB" "All" "Capcity" "lastmodify"
for e in $entry; do
	ossutil64 ls $e |awk -v e=$e -f fenxi.awk
done

time_end=$(date +%s)
time_use=$((time_end - time_start))
echo
echo elapsed time:  $(( time_use / 3600 )) hours $(( time_use % 3600 / 60)) mins $(( time_use % 60)) s

#!/bin/bash

# 分析目标下的所有子目录

target=$1

time_start=$(date +%s)

entry=$(ossutil64 ls -d $target | grep "^oss://")

printf "%-32s%16s%16s%16s%16s%16s%10s%9s%20s\n" "DIR" "<100KB" "<300KB" "<1MB" "<5MB" ">5MB" "All" "Capcity" "lastmodify"

for e in $entry; do

ossutil64 ls $e |awk -v e=$e -f fenxi.awk

done

time_end=$(date +%s)

time_use=$((time_end - time_start))

echo

echo elapsed time: $(( time_use / 3600 )) hours $(( time_use % 3600 / 60)) mins $(( time_use % 60)) s

function format_capcity(num) {
    KB=1024;
    MB=KB*1024;
    GB=MB*1024;
    TB=GB*1024;
    if(num > TB) return sprintf("%6.2fTB", num/TB);
    if(num > GB) return sprintf("%6.2fGB", num/GB);
    if(num > MB) return sprintf("%6.2fMB", num/MB);
    if(num > KB) return sprintf("%6.2fKB", num/KB);
    return sprintf("%6d", num);
}
 
BEGIN{
    k100=0;k300=0;m1=0;m5=0;marge=0;all=0;lastmodify=0;capcity_all=0;
}
 
NR>1 && $8 ~ /^oss:/ {
    all++;
    capcity_all+=$5;
    if($5 < 102400) k100++;
    else if($5 < 1024*300) k300++;
    else if($5 < 1024*1000) {m1++;}
    else if ($5 < 1024 * 1024 * 5) {m5++;}
    else large++;
    if (lastmodify < $1"T"$2) lastmodify=$1"T"$2;
}
 
END{
    gsub("oss://[^/]+", "", e)
    printf("%-32s%9s(%4.1f%%)%9s(%4.1f%%)%9s(%4.1f%%)%9s(%4.1f%%)%9s(%4.1f%%)%10s%9s%20s\n", e, k100, k100*100/all,k300, k300*100/all,m1, m1*100/all,m5, m5*100/all,large, large*100/all, all, format_capcity(capcity_all), lastmodify);
}

function format_capcity(num) {

KB=1024;

MB=KB*1024;

GB=MB*1024;

TB=GB*1024;

if(num > TB) return sprintf("%6.2fTB", num/TB);

if(num > GB) return sprintf("%6.2fGB", num/GB);

if(num > MB) return sprintf("%6.2fMB", num/MB);

if(num > KB) return sprintf("%6.2fKB", num/KB);

return sprintf("%6d", num);

}

BEGIN{

k100=0;k300=0;m1=0;m5=0;marge=0;all=0;lastmodify=0;capcity_all=0;

}

NR>1 && $8 ~ /^oss:/ {

all++;

capcity_all+=$5;

if($5 < 102400) k100++;

else if($5 < 1024*300) k300++;

else if($5 < 1024*1000) {m1++;}

else if ($5 < 1024 * 1024 * 5) {m5++;}

else large++;

if (lastmodify < $1"T"$2) lastmodify=$1"T"$2;

}

END{

gsub("oss://[^/]+", "", e)

printf("%-32s%9s(%4.1f%%)%9s(%4.1f%%)%9s(%4.1f%%)%9s(%4.1f%%)%9s(%4.1f%%)%10s%9s%20s\n", e, k100, k100*100/all,k300, k300*100/all,m1, m1*100/all,m5, m5*100/all,large, large*100/all, all, format_capcity(capcity_all), lastmodify);

}

命令：

DIR                                       <100KB          <300KB            <1MB            <5MB            >5MB       All  Capcity          lastmodify
/dir1/                            2602(16.8%)     7584(49.0%)     3752(24.2%)     1548(10.0%)        2( 0.0%)     15488  11.34GB 2016-06-07T16:15:10
/dir2/                           26( 1.2%)        0( 0.0%)        1( 0.0%)       61( 2.9%)     2034(95.9%)      2122  26.09TB 2018-01-26T18:08:38
/dir3/                                    1(10.0%)        1(10.0%)        2(20.0%)        2(20.0%)        4(40.0%)        10 235.13MB 2016-02-22T15:03:58
/dir4/                                  0( 0.0%)        0( 0.0%)        0( 0.0%)        0( 0.0%)        2(100.0%)         2 396.21MB 2017-03-09T10:17:27

elapsed time: 0 hours 0 mins 3 s

DIR <100KB <300KB <1MB <5MB >5MB All Capcity lastmodify

/dir1/ 2602(16.8%) 7584(49.0%) 3752(24.2%) 1548(10.0%) 2( 0.0%) 15488 11.34GB 2016-06-07T16:15:10

/dir2/ 26( 1.2%) 0( 0.0%) 1( 0.0%) 61( 2.9%) 2034(95.9%) 2122 26.09TB 2018-01-26T18:08:38

/dir3/ 1(10.0%) 1(10.0%) 2(20.0%) 2(20.0%) 4(40.0%) 10 235.13MB 2016-02-22T15:03:58

/dir4/ 0( 0.0%) 0( 0.0%) 0( 0.0%) 0( 0.0%) 2(100.0%) 2 396.21MB 2017-03-09T10:17:27

elapsed time: 0 hours 0 mins 3 s

注意：

结果如果要按照空格分域粘贴到Excel中，需要将百分数前面可能存在的空格给替换掉：

sed s/\(\ /\(/g

1

sed s/\(\ /\(/g

awk 知识点：

自定义函数
内置函数gsub使用
printf格式化函数，sprintf函数
awk 中字符串连接的语法

留下评论

取消回复