Skip to content

Commit

Permalink
updated hdfs-du-viz.sh
Browse files Browse the repository at this point in the history
  • Loading branch information
HariSekhon committed Jul 31, 2019
1 parent 4220041 commit c5108d1
Showing 1 changed file with 7 additions and 7 deletions.
14 changes: 7 additions & 7 deletions contrib/hdfs-du-viz.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,22 @@

max_depth=5

largest_root_dirs=$(hdfs dfs -du -s '/*' | sort -nr | perl -ane 'print "$F[1] "')
largest_root_dirs="$(hdfs dfs -du -s '/*' | sort -nr | perl -ane 'print "$F[1] "')"

printf "%15s %s\n" "bytes" "directory"
for ld in $largest_root_dirs; do
printf "%15.0f %s\n" $(hdfs dfs -du -s $ld| cut -d' ' -f1) $ld
all_dirs=$(hdfs dfs -ls -R $ld | egrep '^dr........' | perl -ane "scalar(split('/',\$_)) <= $max_depth && print \"\$F[7]\n\"" )
printf "%15.0f %s\n" "$(hdfs dfs -du -s "$ld" | cut -d' ' -f1)" "$ld"
all_dirs="$(hdfs dfs -ls -R "$ld" | grep -E '^dr........' | perl -ane "scalar(split('/',\$_)) <= $max_depth && print \"\$F[7]\n\"" )"

for d in $all_dirs; do
line=$(hdfs dfs -du -s $d)
size=$(echo $line | cut -d' ' -f1)
line="$(hdfs dfs -du -s "$d")"
size="$(cut -d' ' -f1 <<< "$line")"
parent_dir=${d%/*}
child=${d##*/}
if [ -n "$parent_dir" ]; then
leading_dirs=$(echo $parent_dir | perl -pe 's/./-/g; s/^.(.+)$/\|$1/')
leading_dirs=$(perl -pe 's/./-/g; s/^.(.+)$/\|$1/' <<< "$parent_dir")
d=${leading_dirs}/$child
fi
printf "%15.0f %s\n" $size $d
printf "%15.0f %s\n" "$size" "$d"
done
done

0 comments on commit c5108d1

Please sign in to comment.