trace-resources: add disk and net stats

trace-resources/src/Cardano/Logging/Resources/Darwin.hsc

@@ -100,6 +100,10 @@ readResourceStatsInternal = getProcessID >>= \pid -> do
     , rHeap       = GhcStats.gcdetails_mem_in_use_bytes $ GhcStats.gc rts
     , rRSS        = _resident_size mem
     , rCentiBlkIO = 0
+    , rNetBytesRd = 0
+    , rNetBytesWr = 0
+    , rFsBytesRd  = 0
+    , rFsBytesWr  = 0
     , rThreads    = 0
     }
  where

trace-resources/src/Cardano/Logging/Resources/Dummy.hs

@@ -25,6 +25,10 @@ readResourceStatsInternal = do
     , rRSS        = 0
     , rHeap       = 0
     , rCentiBlkIO = 0
+    , rNetBytesRd = 0
+    , rNetBytesWr = 0
+    , rFsBytesRd  = 0
+    , rFsBytesWr  = 0
     , rThreads    = 0
     }
  where

trace-resources/src/Cardano/Logging/Resources/Linux.hs

@@ -1,4 +1,6 @@
 {-# LANGUAGE CPP #-}
+{-# LANGUAGE RecordWildCards #-}
+{-# LANGUAGE TypeApplications #-}
 
 module Cardano.Logging.Resources.Linux
     (
@@ -14,20 +16,103 @@ import           System.Posix.Files              (fileMode, getFileStatus,
                                                   ownerReadMode)
 import           Text.Read                       (readMaybe)
 
+-- * Disk IO stats:
+-- /proc/[pid]/io (since kernel 2.6.20)
+--        This file contains I/O statistics for the process, for example:
+--
+--               # cat /proc/3828/io
+--               rchar: 323934931
+--               wchar: 323929600
+--               syscr: 632687
+--               syscw: 632675
+--               read_bytes: 0
+--               write_bytes: 323932160
+--               cancelled_write_bytes: 0
+--
+--        The fields are as follows:
+--
+--        rchar: characters read
+--               The number of bytes which this task has caused to be read from storage.  This is simply the  sum
+--               of bytes which this process passed to read(2) and similar system calls.  It includes things such
+--               as terminal I/O and is unaffected by whether or not actual physical disk I/O was  required  (the
+--               read might have been satisfied from pagecache).
+--
+--        wchar: characters written
+--               The  number  of bytes which this task has caused, or shall cause to be written to disk.  Similar
+--               caveats apply here as with rchar.
+--
+--        syscr: read syscalls
+--               Attempt to count the number of read I/O operations-that is, system calls  such  as  read(2)  and
+--               pread(2).
+--
+--        syscw: write syscalls
+--               Attempt  to  count the number of write I/O operations-that is, system calls such as write(2) and
+--               pwrite(2).
+--
+--        read_bytes: bytes read
+--               Attempt to count the number of bytes which this process really did cause to be fetched from  the
+--               storage layer.  This is accurate for block-backed filesystems.
+--
+--        write_bytes: bytes written
+--               Attempt to count the number of bytes which this process caused to be sent to the storage layer.
+--
+--        cancelled_write_bytes:
+--               The  big  inaccuracy  here  is truncate.  If a process writes 1MB to a file and then deletes the
+--               file, it will in fact perform no writeout.  But it will have been accounted as having caused 1MB
+--               of  write.   In other words: this field represents the number of bytes which this process caused
+--               to not happen, by truncating pagecache.  A task can cause "negative"  I/O  too.   If  this  task
+--               truncates  some  dirty  pagecache,  some  I/O  which another task has been accounted for (in its
+--               write\_bytes) will not be happening.
+--
+--        Note: In the current implementation, things are a bit racy  on  32-bit  systems:  if  process  A  reads
+--        process  B's  /proc/[pid]/io  while process B is updating one of these 64-bit counters, process A could
+--        see an intermediate result.
+--
+--        Permission to access this file is governed by a ptrace access mode PTRACE\_MODE\_READ\_FSCREDS check;  see
+--        ptrace(2).
+--
+readProcBlockInOut :: IO (Word64, Word64)
+readProcBlockInOut = do
+    fields <- readProcList "/proc/self/io"
+    case -- We're only interested in 'read_bytes' & 'write_bytes':
+      fmap fromInteger . take 3 . drop 9 $ fields of
+      fsRd: _:fsWr:[] -> pure (fsRd, fsWr)
+      _ -> pure (0, 0)
+
+-- * Network stats:
+--   grep IpExt /proc/<pid>/net/netstat
+--   IpExt: InNoRoutes InTruncatedPkts InMcastPkts OutMcastPkts InBcastPkts OutBcastPkts InOctets OutOctets InMcastOctets OutMcastOctets InBcastOctets OutBcastOctets InCsumErrors InNoECTPkts InECT1Pkts InECT0Pkts InCEPkts
+--   IpExt: 0 0 20053 8977 2437 23 3163525943 196480057 2426648 1491754 394285 5523 0 3513269 0 217426 0
+--
+readProcNetInOut :: IO (Word64, Word64)
+readProcNetInOut = do
+    fields <- words . lastline . lines <$> readFile "/proc/self/net/netstat"
+    case -- We're only interested in 'InOctets' & 'OutOctets':
+      fmap readMaybe . take 2 . drop 7 $ fields of
+      Just netIn:Just netOut:[] -> pure (netIn, netOut)
+      _ -> pure (0, 0)
+ where
+   lastline ls | length ls == 4 = last ls -- ensures we read the fourth line
+               | otherwise = []
+
 -- | TODO we have to expand the |readMemStats| function
 -- to read full data from |proc|
 readResourceStatsInternal :: IO (Maybe ResourceStats)
 readResourceStatsInternal = do
   rts <- GhcStats.getRTSStats
-  mkProcStats rts . fmap fromIntegral <$> readProcList "/proc/self/stat"
+  net <- readProcNetInOut
+  fs  <- readProcBlockInOut
+  mkProcStats rts net fs . fmap fromIntegral <$> readProcList "/proc/self/stat"
  where
-   mkProcStats :: GhcStats.RTSStats -> [Word64] -> Maybe ResourceStats
+   mkProcStats :: GhcStats.RTSStats -> (Word64, Word64) -> (Word64, Word64) -> [Word64] -> Maybe ResourceStats
    mkProcStats rts
-               (_:_:_:_:_:_:_:_:_:_            -- 00-09
-               :_:_:_:user:sys:_:_:_:_:threads -- 10-19
-               :_:_:_:rss:_:_:_:_:_:_          -- 20-29
-               :_:_:_:_:_:_:_:_:_:_            -- 30-39
-               :_:blkio:_rest) =               -- 40-42
+               (rNetBytesRd, rNetBytesWr)
+               (rFsBytesRd,  rFsBytesWr)
+               (_:_:_:_:_:_:_:_:_:_             -- 00-09
+               :_:_:_:user:sys:_:_:_:_:rThreads -- 10-19
+               :_:_:_:rss:_:_:_:_:_:_           -- 20-29
+               :_:_:_:_:_:_:_:_:_:_             -- 30-39
+               :_:rCentiBlkIO:_rest) =          -- 40-42
      Just $ Resources
        { rCentiCpu   = user + sys
        , rCentiGC    = nsToCenti $ GhcStats.gc_cpu_ns rts
@@ -38,10 +123,9 @@ readResourceStatsInternal = do
        , rLive       = GhcStats.gcdetails_live_bytes $ GhcStats.gc rts
        , rHeap       = GhcStats.gcdetails_mem_in_use_bytes $ GhcStats.gc rts
        , rRSS        = rss * 4096 -- TODO:  this is really PAGE_SIZE.
-       , rCentiBlkIO = blkio
-       , rThreads    = threads
+       , ..
        }
-   mkProcStats _ _ = Nothing
+   mkProcStats _ _ _ _ = Nothing
    nsToCenti :: GhcStats.RtsTime -> Word64
    nsToCenti = floor . (/ (10000000 :: Double)) . fromIntegral
 

trace-resources/src/Cardano/Logging/Resources/Types.hs

@@ -2,6 +2,7 @@
 {-# LANGUAGE DeriveFunctor #-}
 {-# LANGUAGE DeriveGeneric #-}
 {-# LANGUAGE FlexibleInstances #-}
+{-# LANGUAGE RecordWildCards #-}
 
 module Cardano.Logging.Resources.Types
     ( Resources(..)
@@ -33,12 +34,16 @@ data Resources a
       , rHeap       :: !a
       , rRSS        :: !a
       , rCentiBlkIO :: !a
+      , rNetBytesRd :: !a
+      , rNetBytesWr :: !a
+      , rFsBytesRd  :: !a
+      , rFsBytesWr  :: !a
       , rThreads    :: !a
       }
   deriving (Functor, Generic, Show)
 
 instance Applicative Resources where
-  pure a = Resources a a a a a a a a a a a
+  pure a = Resources a a a a a a a a a a a a a a a
   f <*> x =
     Resources
     { rCentiCpu   = rCentiCpu   f (rCentiCpu   x)
@@ -51,6 +56,10 @@ instance Applicative Resources where
     , rHeap       = rHeap       f (rHeap       x)
     , rRSS        = rRSS        f (rRSS        x)
     , rCentiBlkIO = rCentiBlkIO f (rCentiBlkIO x)
+    , rNetBytesRd = rNetBytesRd f (rNetBytesRd x)
+    , rNetBytesWr = rNetBytesWr f (rNetBytesWr x)
+    , rFsBytesRd  = rFsBytesRd  f (rFsBytesRd  x)
+    , rFsBytesWr  = rFsBytesWr  f (rFsBytesWr  x)
     , rThreads    = rThreads    f (rThreads    x)
     }
 
@@ -76,30 +85,38 @@ docResourceStats :: Documented ResourceStats
 docResourceStats = Documented [
       DocMsg
         []
-        [("Resources.Stat.Cputicks", "Reports the CPU ticks, sice the process was started")
-        ,("Resources.Mem.Resident", "")
-        ,("Resources.RTS.GcLiveBytes", "")
-        ,("Resources.RTS.GcMajorNum", "")
-        ,("Resources.RTS.GcMinorNum", "")
-        ,("Resources.RTS.Gcticks", "")
-        ,("Resources.RTS.Mutticks", "")
-        ,("Resources.RTS.Threads","")
+        [("Resources.Stat.Cputicks", "Kernel-reported CPU ticks (1/100th of a second), since process start")
+        ,("Resources.Mem.Resident", "Kernel-reported RSS (resident set size)")
+        ,("Resources.RTS.GcLiveBytes", "RTS-reported live bytes")
+        ,("Resources.RTS.GcMajorNum", "Major GCs")
+        ,("Resources.RTS.GcMinorNum", "Minor GCs")
+        ,("Resources.RTS.Gcticks", "RTS-reported CPU ticks spent on GC")
+        ,("Resources.RTS.Mutticks", "RTS-reported CPU ticks spent on mutator")
+        ,("Resources.State.NetBytesRd", "IP packet bytes read")
+        ,("Resources.State.NetBytesWr", "IP packet bytes written")
+        ,("Resources.State.FsBytesRd", "FS bytes read")
+        ,("Resources.State.FsBytesWr", "FS bytes written")
+        ,("Resources.RTS.Threads","RTS green thread count")
         ]
         ""
     ]
 
 instance LogFormatting ResourceStats where
-    forHuman rs = "Resources:"
-                  <>  " Cpu Ticks "            <> (pack . show) (rCentiCpu rs)
-                  <> ", GC centiseconds "      <> (pack . show) (rCentiGC rs)
-                  <> ", Mutator centiseconds " <> (pack . show) (rCentiMut rs)
-                  <> ", GCs major "            <> (pack . show) (rGcsMajor rs)
-                  <> ", GCs minor "            <> (pack . show) (rGcsMinor rs)
-                  <> ", Allocated bytes "      <> (pack . show) (rAlloc rs)
-                  <>" , GC live bytes "        <> (pack . show) (rLive rs)
-                  <> ", RTS heap "             <> (pack . show) (rHeap rs)
-                  <> ", RSS "                  <> (pack . show) (rRSS rs)
-                  <> ", Threads "              <> (pack . show) (rThreads rs)
+    forHuman Resources{..} = "Resources:"
+                  <>  " Cpu Ticks "            <> (pack . show) rCentiCpu
+                  <> ", GC centiseconds "      <> (pack . show) rCentiGC
+                  <> ", Mutator centiseconds " <> (pack . show) rCentiMut
+                  <> ", GCs major "            <> (pack . show) rGcsMajor
+                  <> ", GCs minor "            <> (pack . show) rGcsMinor
+                  <> ", Allocated bytes "      <> (pack . show) rAlloc
+                  <>" , GC live bytes "        <> (pack . show) rLive
+                  <> ", RTS heap "             <> (pack . show) rHeap
+                  <> ", RSS "                  <> (pack . show) rRSS
+                  <> ", Net bytes read "       <> (pack . show) rNetBytesRd
+                  <> " written "               <> (pack . show) rNetBytesWr
+                  <> ", FS bytes read "        <> (pack . show) rFsBytesRd
+                  <> " written "               <> (pack . show) rFsBytesWr
+                  <> ", Threads "              <> (pack . show) rThreads
                   <> "."
 
     forMachine _dtal rs = mconcat
@@ -114,16 +131,27 @@ instance LogFormatting ResourceStats where
       , "Heap"          .= Number (fromIntegral $ rHeap rs)
       , "RSS"           .= Number (fromIntegral $ rRSS rs)
       , "CentiBlkIO"    .= Number (fromIntegral $ rCentiBlkIO rs)
+      , "NetBytesRd"    .= Number (fromIntegral $ rNetBytesRd rs)
+      , "NetBytesWr"    .= Number (fromIntegral $ rNetBytesWr rs)
+      , "FsBytesRd"     .= Number (fromIntegral $ rFsBytesRd rs)
+      , "FsBytesWr"     .= Number (fromIntegral $ rFsBytesWr rs)
       , "Threads"       .= Number (fromIntegral $ rThreads rs)
       ]
 
     asMetrics rs =
-      [ IntM "Resources.Stat.Cputicks" (fromIntegral $ rCentiCpu rs)
-      , IntM "Resources.Mem.Resident" (fromIntegral $ rRSS rs)
-      , IntM "Resources.RTS.GcLiveBytes" (fromIntegral $ rLive rs)
-      , IntM "Resources.RTS.GcMajorNum" (fromIntegral $ rGcsMajor rs)
-      , IntM "Resources.RTS.GcMinorNum" (fromIntegral $ rGcsMinor rs)
-      , IntM "Resources.RTS.Gcticks" (fromIntegral $ rCentiGC rs)
-      , IntM "Resources.RTS.Mutticks" (fromIntegral $ rCentiMut rs)
+      [ IntM "Resources.Stat.Cputicks"    (fromIntegral $ rCentiCpu rs)
+      , IntM "Resources.RTS.Gcticks"      (fromIntegral $ rCentiGC rs)
+      , IntM "Resources.RTS.Mutticks"     (fromIntegral $ rCentiMut rs)
+      , IntM "Resources.RTS.GcMajorNum"   (fromIntegral $ rGcsMajor rs)
+      , IntM "Resources.RTS.GcMinorNum"   (fromIntegral $ rGcsMinor rs)
+      , IntM "Resources.RTS.Alloc"        (fromIntegral $ rAlloc rs)
+      , IntM "Resources.RTS.GcLiveBytes"  (fromIntegral $ rLive rs)
+      , IntM "Resources.RTS.Heap"         (fromIntegral $ rHeap rs)
+      , IntM "Resources.Mem.Resident"     (fromIntegral $ rRSS rs)
+      , IntM "Resources.Stat.BlkIOticks"  (fromIntegral $ rCentiBlkIO rs)
+      , IntM "Resources.State.NetBytesRd" (fromIntegral $ rNetBytesRd rs)
+      , IntM "Resources.State.NetBytesWr" (fromIntegral $ rNetBytesWr rs)
+      , IntM "Resources.State.FsBytesRd"  (fromIntegral $ rFsBytesRd rs)
+      , IntM "Resources.State.FsBytesWr"  (fromIntegral $ rFsBytesWr rs)
       , IntM "Resources.RTS.Stat.Threads" (fromIntegral $ rThreads rs)
       ]

trace-resources/src/Cardano/Logging/Resources/Windows.hsc

@@ -154,6 +154,10 @@ readResourceStatsInternal = getCurrentProcessId >>= \pid -> do
     , rHeap       = GhcStats.gcdetails_mem_in_use_bytes $ GhcStats.gc rts
     , rRSS        = fromIntegral (_workingSetSize mem)
     , rCentiBlkIO = 0
+    , rNetBytesRd = 0
+    , rNetBytesWr = 0
+    , rFsBytesRd  = 0
+    , rFsBytesWr  = 0
     , rThreads    = 0
     }
  where